Beispiel #1
0
 def resolvePath(self, value, mustExist, errorMsg):
     try:
         return utils.resolvePath(
             value, self._configView.pathDict.get('search_paths', []),
             mustExist, ConfigError)
     except Exception:
         raise ConfigError(errorMsg)
	def _fillContentWithIncludes(self, configFile, searchPaths, configContent):
		log = logging.getLogger(('config.%s' % utils.getRootName(configFile)).rstrip('.').lower())
		log.log(logging.INFO1, 'Reading config file %s', configFile)
		configFile = utils.resolvePath(configFile, searchPaths, ErrorClass = ConfigError)
		configFileLines = SafeFile(configFile).readlines()

		# Single pass, non-recursive list retrieval
		tmpConfigContent = {}
		self._fillContentSingleFile(configFile, configFileLines, searchPaths, tmpConfigContent)
		def getFlatList(section, option):
			for (opt, value, src) in tmpConfigContent.get(section, []):
				try:
					if opt == option:
						for entry in parseList(value, None):
							yield entry
				except Exception:
					raise ConfigError('Unable to parse [%s] %s from %s' % (section, option, src))

		newSearchPaths = [os.path.dirname(configFile)]
		# Add entries from include statement recursively
		for includeFile in getFlatList('global', 'include'):
			self._fillContentWithIncludes(includeFile, searchPaths + newSearchPaths, configContent)
		# Process all other entries in current file
		self._fillContentSingleFile(configFile, configFileLines, searchPaths, configContent)
		# Override entries in current config file
		for overrideFile in getFlatList('global', 'include override'):
			self._fillContentWithIncludes(overrideFile, searchPaths + newSearchPaths, configContent)
		# Filter special global options
		if configContent.get('global', []):
			configContent['global'] = lfilter(lambda opt_v_s: opt_v_s[0] not in ['include', 'include override'], configContent['global'])
		return searchPaths + newSearchPaths
Beispiel #3
0
	def _fillContentWithIncludes(self, configFile, searchPaths, configContent):
		log = logging.getLogger(('config.%s' % utils.getRootName(configFile)).rstrip('.').lower())
		log.log(logging.INFO1, 'Reading config file %s', configFile)
		configFile = utils.resolvePath(configFile, searchPaths, ErrorClass = ConfigError)
		configFileLines = SafeFile(configFile).readlines()

		# Single pass, non-recursive list retrieval
		tmpConfigContent = {}
		self._fillContentSingleFile(configFile, configFileLines, searchPaths, tmpConfigContent)
		def getFlatList(section, option):
			for (opt, value, src) in tmpConfigContent.get(section, []):
				try:
					if opt == option:
						for entry in parseList(value, None):
							yield entry
				except Exception:
					raise ConfigError('Unable to parse [%s] %s from %s' % (section, option, src))

		newSearchPaths = [os.path.dirname(configFile)]
		# Add entries from include statement recursively
		for includeFile in getFlatList('global', 'include'):
			self._fillContentWithIncludes(includeFile, searchPaths + newSearchPaths, configContent)
		# Process all other entries in current file
		self._fillContentSingleFile(configFile, configFileLines, searchPaths, configContent)
		# Override entries in current config file
		for overrideFile in getFlatList('global', 'include override'):
			self._fillContentWithIncludes(overrideFile, searchPaths + newSearchPaths, configContent)
		# Filter special global options
		if configContent.get('global', []):
			configContent['global'] = lfilter(lambda opt_v_s: opt_v_s[0] not in ['include', 'include override'], configContent['global'])
		return searchPaths + newSearchPaths
	def queryDBSv3(self, api, **params):
		if not os.environ.get('X509_USER_PROXY', ''):
			raise UserError('VOMS proxy needed to query DBS3! Environment variable X509_USER_PROXY is not set!')
		proxyPath = utils.resolvePath(os.environ.get('X509_USER_PROXY', ''), mustExist = False)
		if not os.path.exists(proxyPath):
			raise UserError('VOMS proxy needed to query DBS3! Environment variable X509_USER_PROXY is "%s"' % proxyPath)
		return readJSON(self.url + '/%s' % api, params, cert = proxyPath)
Beispiel #5
0
	def _parseFile(self, container, configFile, defaults = None, searchPaths = []):
		try:
			configFile = utils.resolvePath(configFile, searchPaths, ErrorClass = ConfigError)
			log = logging.getLogger(('config.%s' % utils.getRootName(configFile)).rstrip('.'))
			log.log(logging.INFO1, 'Reading config file %s' % configFile)
			for line in map(lambda x: x.rstrip() + '=:', open(configFile, 'r').readlines()):
				if line.startswith('[') or line.lstrip().startswith(';'):
					continue # skip section and comment lines
				# Abort if non-indented line with ":" preceeding "=" was found
				if (line.lstrip() == line) and (line.find(":") < line.find("=")):
					raise ConfigError('Invalid config line:\n\t%s\nPlease use "key = value" syntax or indent values!' % line)
			# Parse with python config parser
			parser = ConfigParser.ConfigParser(defaults)
			parser.readfp(open(configFile, 'r'))
			# Parse include files
			if parser.has_option('global', 'include'):
				includeFiles = parser.get('global', 'include').split('#')[0].split(';')[0]
				for includeFile in utils.parseList(includeFiles, None):
					self._parseFile(container, includeFile, parser.defaults(),
						searchPaths + [os.path.dirname(configFile)])
			# Store config settings
			for section in parser.sections():
				for option in parser.options(section):
					if (section, option) != ('global', 'include'):
						value_list = parser.get(section, option).splitlines() # Strip comments
						value_list = map(lambda l: l.rsplit(';', 1)[0].strip(), value_list)
						value_list = filter(lambda l: l != '', value_list)
						container.setEntry(section, option, str.join('\n', value_list), configFile)
		except:
			raise RethrowError('Error while reading configuration file "%s"!' % configFile, ConfigError)
    def __init__(self):
        # Collect host / user / installation specific config files
        def resolve_hostname():
            import socket
            host = socket.gethostname()
            try:
                return socket.gethostbyaddr(host)[0]
            except Exception:
                return host

        try:
            host = hang_protection(resolve_hostname, timeout=5)
            hostCfg = lmap(
                lambda c: utils.pathPKG('../config/%s.conf' % host.split(
                    '.', c)[-1]), irange(host.count('.') + 1, -1, -1))
        except TimeoutException:
            sys.stderr.write('System call to resolve hostname is hanging!\n')
            sys.stderr.flush()
            hostCfg = []
        defaultCfg = [
            '/etc/grid-control.conf', '~/.grid-control.conf',
            utils.pathPKG('../config/default.conf')
        ]
        if os.environ.get('GC_CONFIG'):
            defaultCfg.append('$GC_CONFIG')
        log = logging.getLogger('config.default')
        log.log(logging.DEBUG1, 'Possible default config files: %s',
                str.join(', ', defaultCfg))
        fqConfigFiles = lmap(lambda p: utils.resolvePath(p, mustExist=False),
                             hostCfg + defaultCfg)
        FileConfigFiller.__init__(self,
                                  lfilter(os.path.exists, fqConfigFiles),
                                  addSearchPath=False)
Beispiel #7
0
    def __init__(self, filler=None, configFilePath=None):
        def getName(prefix=''):
            if configFilePath:
                return ('%s.%s' %
                        (prefix, utils.getRootName(configFilePath))).strip('.')
            elif prefix:
                return prefix
            return 'unnamed'

        pathMain = os.getcwd()
        if configFilePath:
            pathMain = os.path.dirname(
                utils.resolvePath(configFilePath,
                                  searchPaths=[os.getcwd()],
                                  ErrorClass=ConfigError))

        # Init config containers
        self._curContainer = ConfigContainer('current')
        if filler:  # Read in the current configuration ...
            filler.fill(self._curContainer)
        logging.getLogger('config.stored').propagate = False
        oldContainer = ConfigContainer('stored')
        oldContainer.enabled = False

        # Create config view and temporary config interface
        self._view = SimpleConfigView(getName(), oldContainer,
                                      self._curContainer)
        self._view.pathDict['search_paths'] = UniqueList(
            [os.getcwd(), pathMain])

        # Determine work directory using config interface with "global" scope
        tmpInterface = SimpleConfigInterface(
            self._view.getView(setSections=['global']))
        wdBase = tmpInterface.getPath('workdir base',
                                      pathMain,
                                      mustExist=False)
        pathWork = tmpInterface.getPath('workdir',
                                        os.path.join(wdBase, getName('work')),
                                        mustExist=False)
        self._view.pathDict[
            '<WORKDIR>'] = pathWork  # tmpInterface still has undefinied
        # Set dynamic plugin search path
        sys.path.extend(tmpInterface.getPaths('plugin paths', [os.getcwd()]))

        # Determine and load stored config settings
        self._flatCfgPath = os.path.join(pathWork,
                                         'current.conf')  # Minimal config file
        self._oldCfgPath = os.path.join(
            pathWork, 'work.conf')  # Config file with saved settings
        if os.path.exists(self._oldCfgPath):
            GeneralFileConfigFiller([self._oldCfgPath]).fill(oldContainer)
            CompatConfigFiller(os.path.join(pathWork,
                                            'task.dat')).fill(oldContainer)
            oldContainer.enabled = True
            oldContainer.setReadOnly()

        # Get persistent variables - only possible after oldContainer was enabled
        self._view.setConfigName(
            tmpInterface.get('config id', getName(), persistent=True))
Beispiel #8
0
	def __init__(self):
		# Collect host / user / installation specific config files
		host = socket.gethostbyaddr(socket.gethostname())[0]
		hostCfg = map(lambda c: utils.pathGC('config/%s.conf' % host.split('.', c)[-1]), range(host.count('.') + 1, 0, -1))
		defaultCfg = ['/etc/grid-control.conf', '~/.grid-control.conf', utils.pathGC('config/default.conf')]
		if os.environ.get('GC_CONFIG'):
			defaultCfg.append('$GC_CONFIG')
		fqConfigFiles = map(lambda p: utils.resolvePath(p, mustExist = False), hostCfg + defaultCfg)
		FileConfigFiller.__init__(self, filter(os.path.exists, fqConfigFiles))
	def _fillContentFromFile(self, configFile, searchPaths, configContent = {}):
		log = logging.getLogger(('config.%s' % utils.getRootName(configFile)).rstrip('.'))
		log.log(logging.INFO1, 'Reading config file %s' % configFile)
		configFile = utils.resolvePath(configFile, searchPaths, ErrorClass = ConfigError)
		configFileData = open(configFile, 'r').readlines()

		# Single pass, non-recursive list retrieval
		tmpConfigContent = {}
		self._fillContentFromSingleFile(configFile, configFileData, searchPaths, tmpConfigContent)
		def getFlatList(section, option):
			for (opt, value, s) in filter(lambda (opt, v, s): opt == option, tmpConfigContent.get(section, [])):
				for entry in utils.parseList(value, None):
					yield entry
	def __init__(self, filler = None, configFilePath = None):
		def getName(prefix = ''):
			if configFilePath:
				return ('%s.%s' % (prefix, utils.getRootName(configFilePath))).strip('.')
			elif prefix:
				return prefix
			return 'unnamed'

		try:
			pathMain = os.getcwd()
		except Exception:
			raise ConfigError('The current directory does not exist!')
		if configFilePath:
			pathMain = os.path.dirname(utils.resolvePath(configFilePath,
				searchPaths = [os.getcwd()], ErrorClass = ConfigError))

		# Init config containers
		self._curContainer = ConfigContainer('current')
		if filler: # Read in the current configuration ...
			filler.fill(self._curContainer)
		self._curContainer.resolve() # resolve interpolations

		logging.getLogger('config.stored').propagate = False
		oldContainer = ConfigContainer('stored')
		oldContainer.enabled = False

		# Create config view and temporary config interface
		self._view = SimpleConfigView(getName(), oldContainer, self._curContainer)
		self._view.pathDict['search_paths'] = UniqueList([os.getcwd(), pathMain])

		# Determine work directory using config interface with "global" scope
		tmp_config = SimpleConfigInterface(self._view.getView(setSections = ['global']))
		wdBase = tmp_config.getPath('workdir base', pathMain, mustExist = False)
		pathWork = tmp_config.getPath('workdir', os.path.join(wdBase, getName('work')), mustExist = False)
		self._view.pathDict['<WORKDIR>'] = pathWork # tmp_config still has undefinied
		# Set dynamic plugin search path
		sys.path.extend(tmp_config.getPaths('plugin paths', [os.getcwd()]))

		# Determine and load stored config settings
		self._flatCfgPath = os.path.join(pathWork, 'current.conf') # Minimal config file
		self._oldCfgPath = os.path.join(pathWork, 'work.conf') # Config file with saved settings
		if os.path.exists(self._oldCfgPath):
			GeneralFileConfigFiller([self._oldCfgPath]).fill(oldContainer)
			CompatConfigFiller(os.path.join(pathWork, 'task.dat')).fill(oldContainer)
			oldContainer.enabled = True
			oldContainer.setReadOnly()

		# Get persistent variables - only possible after oldContainer was enabled
		self._view.setConfigName(tmp_config.get('config id', getName(), persistent = True))
Beispiel #11
0
    def _getSandboxFiles(self, task, monitor, smList):
        # Prepare all input files
        depList = set(
            ichain(imap(lambda x: x.getDependencies(), [task] + smList)))
        depPaths = lmap(lambda pkg: utils.pathShare('', pkg=pkg),
                        os.listdir(utils.pathPKG()))
        depFiles = lmap(
            lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths),
            depList)
        taskEnv = utils.mergeDicts(
            imap(lambda x: x.getTaskConfig(), [monitor, task] + smList))
        taskEnv.update({
            'GC_DEPFILES': str.join(' ', depList),
            'GC_USERNAME': self._token.getUsername(),
            'GC_WMS_NAME': self.wmsName
        })
        taskConfig = sorted(
            utils.DictFormat(escapeString=True).format(
                taskEnv, format='export %s%s%s\n'))
        varMappingDict = dict(
            izip(monitor.getTaskConfig().keys(),
                 monitor.getTaskConfig().keys()))
        varMappingDict.update(task.getVarMapping())
        varMapping = sorted(
            utils.DictFormat(delimeter=' ').format(varMappingDict,
                                                   format='%s%s%s\n'))

        # Resolve wildcards in task input files
        def getTaskFiles():
            for f in task.getSBInFiles():
                matched = glob.glob(f.pathAbs)
                if matched != []:
                    for match in matched:
                        yield match
                else:
                    yield f.pathAbs

        return lchain([
            monitor.getFiles(), depFiles,
            getTaskFiles(),
            [
                VirtualFile('_config.sh', taskConfig),
                VirtualFile('_varmap.dat', varMapping)
            ]
        ])
	def __init__(self):
		# Collect host / user / installation specific config files
		def resolve_hostname():
			import socket
			host = socket.gethostname()
			try:
				return socket.gethostbyaddr(host)[0]
			except Exception:
				return host
		try:
			host = hang_protection(resolve_hostname, timeout = 5)
			hostCfg = lmap(lambda c: utils.pathPKG('../config/%s.conf' % host.split('.', c)[-1]), irange(host.count('.') + 1, -1, -1))
		except TimeoutException:
			sys.stderr.write('System call to resolve hostname is hanging!\n')
			sys.stderr.flush()
			hostCfg = []
		defaultCfg = ['/etc/grid-control.conf', '~/.grid-control.conf', utils.pathPKG('../config/default.conf')]
		if os.environ.get('GC_CONFIG'):
			defaultCfg.append('$GC_CONFIG')
		log = logging.getLogger('config.default')
		log.log(logging.DEBUG1, 'Possible default config files: %s', str.join(', ', defaultCfg))
		fqConfigFiles = lmap(lambda p: utils.resolvePath(p, mustExist = False), hostCfg + defaultCfg)
		FileConfigFiller.__init__(self, lfilter(os.path.exists, fqConfigFiles), addSearchPath = False)
Beispiel #13
0
	def _getSandboxFiles(self, task, monitor, smList):
		# Prepare all input files
		depList = set(ichain(imap(lambda x: x.getDependencies(), [task] + smList)))
		depPaths = lmap(lambda pkg: utils.pathShare('', pkg = pkg), os.listdir(utils.pathPKG()))
		depFiles = lmap(lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths), depList)
		taskEnv = utils.mergeDicts(imap(lambda x: x.getTaskConfig(), [monitor, task] + smList))
		taskEnv.update({'GC_DEPFILES': str.join(' ', depList), 'GC_USERNAME': self._token.getUsername(),
			'GC_WMS_NAME': self._name})
		taskConfig = sorted(utils.DictFormat(escapeString = True).format(taskEnv, format = 'export %s%s%s\n'))
		varMappingDict = dict(izip(monitor.getTaskConfig().keys(), monitor.getTaskConfig().keys()))
		varMappingDict.update(task.getVarMapping())
		varMapping = sorted(utils.DictFormat(delimeter = ' ').format(varMappingDict, format = '%s%s%s\n'))
		# Resolve wildcards in task input files
		def getTaskFiles():
			for f in task.getSBInFiles():
				matched = glob.glob(f.pathAbs)
				if matched != []:
					for match in matched:
						yield match
				else:
					yield f.pathAbs
		return lchain([monitor.getFiles(), depFiles, getTaskFiles(),
			[VirtualFile('_config.sh', taskConfig), VirtualFile('_varmap.dat', varMapping)]])
Beispiel #14
0
	def _getSandboxFiles(self, module, monitor, smList):
		# Prepare all input files
		depList = set(itertools.chain(*map(lambda x: x.getDependencies(), [module] + smList)))
		depPaths = map(lambda pkg: utils.pathShare('', pkg = pkg), os.listdir(utils.pathGC('packages')))
		depFiles = map(lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths), depList)
		taskEnv = list(itertools.chain(map(lambda x: x.getTaskConfig(), [monitor, module] + smList)))
		taskEnv.append({'GC_DEPFILES': str.join(' ', depList), 'GC_USERNAME': self.proxy.getUsername(),
			'GC_WMS_NAME': self.wmsName})
		taskConfig = sorted(utils.DictFormat(escapeString = True).format(utils.mergeDicts(taskEnv), format = 'export %s%s%s\n'))
		varMappingDict = dict(zip(monitor.getTaskConfig().keys(), monitor.getTaskConfig().keys()))
		varMappingDict.update(module.getVarMapping())
		varMapping = sorted(utils.DictFormat(delimeter = ' ').format(varMappingDict, format = '%s%s%s\n'))
		# Resolve wildcards in module input files
		def getModuleFiles():
			for f in module.getSBInFiles():
				matched = glob.glob(f)
				if matched != []:
					for match in matched:
						yield match
				else:
					yield f
		return list(itertools.chain(monitor.getFiles(), depFiles, getModuleFiles(),
			[utils.VirtualFile('_config.sh', taskConfig), utils.VirtualFile('_varmap.dat', varMapping)]))
	def resolvePath(self, value, mustExist, errorMsg):
		try:
			return utils.resolvePath(value, self._configView.pathDict.get('search_paths', []), mustExist, ConfigError)
		except Exception:
			raise ConfigError(errorMsg)
Beispiel #16
0
	def resolvePath(self, value, mustExist, errorMsg):
		try:
			return utils.resolvePath(value, [self._pathBase], mustExist, ConfigError)
		except:
			raise RethrowError(errorMsg, ConfigError)