Exemplo n.º 1
0
	def queryDBSv3(self, api, **params):
		if not os.environ.get('X509_USER_PROXY', ''):
			raise UserError('VOMS proxy needed to query DBS3! Environment variable X509_USER_PROXY is not set!')
		proxyPath = utils.resolvePath(os.environ.get('X509_USER_PROXY', ''), mustExist = False)
		if not os.path.exists(proxyPath):
			raise UserError('VOMS proxy needed to query DBS3! Environment variable X509_USER_PROXY is "%s"' % proxyPath)
		return readJSON(self.url + '/%s' % api, params, cert = proxyPath)
Exemplo n.º 2
0
	def _fillContentWithIncludes(self, configFile, searchPaths, configContent):
		log = logging.getLogger(('config.%s' % utils.getRootName(configFile)).rstrip('.').lower())
		log.log(logging.INFO1, 'Reading config file %s', configFile)
		configFile = utils.resolvePath(configFile, searchPaths, ErrorClass = ConfigError)
		configFileLines = SafeFile(configFile).readlines()

		# Single pass, non-recursive list retrieval
		tmpConfigContent = {}
		self._fillContentSingleFile(configFile, configFileLines, searchPaths, tmpConfigContent)
		def getFlatList(section, option):
			for (opt, value, src) in tmpConfigContent.get(section, []):
				try:
					if opt == option:
						for entry in parseList(value, None):
							yield entry
				except Exception:
					raise ConfigError('Unable to parse [%s] %s from %s' % (section, option, src))

		newSearchPaths = [os.path.dirname(configFile)]
		# Add entries from include statement recursively
		for includeFile in getFlatList('global', 'include'):
			self._fillContentWithIncludes(includeFile, searchPaths + newSearchPaths, configContent)
		# Process all other entries in current file
		self._fillContentSingleFile(configFile, configFileLines, searchPaths, configContent)
		# Override entries in current config file
		for overrideFile in getFlatList('global', 'include override'):
			self._fillContentWithIncludes(overrideFile, searchPaths + newSearchPaths, configContent)
		# Filter special global options
		if configContent.get('global', []):
			configContent['global'] = lfilter(lambda opt_v_s: opt_v_s[0] not in ['include', 'include override'], configContent['global'])
		return searchPaths + newSearchPaths
Exemplo n.º 3
0
	def _parseFile(self, container, configFile, defaults = None, searchPaths = []):
		try:
			configFile = utils.resolvePath(configFile, searchPaths, ErrorClass = ConfigError)
			log = logging.getLogger(('config.%s' % utils.getRootName(configFile)).rstrip('.'))
			log.log(logging.INFO1, 'Reading config file %s' % configFile)
			for line in map(lambda x: x.rstrip() + '=:', open(configFile, 'r').readlines()):
				if line.startswith('[') or line.lstrip().startswith(';'):
					continue # skip section and comment lines
				# Abort if non-indented line with ":" preceeding "=" was found
				if (line.lstrip() == line) and (line.find(":") < line.find("=")):
					raise ConfigError('Invalid config line:\n\t%s\nPlease use "key = value" syntax or indent values!' % line)
			# Parse with python config parser
			parser = ConfigParser.ConfigParser(defaults)
			parser.readfp(open(configFile, 'r'))
			# Parse include files
			if parser.has_option('global', 'include'):
				includeFiles = parser.get('global', 'include').split('#')[0].split(';')[0]
				for includeFile in utils.parseList(includeFiles, None):
					self._parseFile(container, includeFile, parser.defaults(),
						searchPaths + [os.path.dirname(configFile)])
			# Store config settings
			for section in parser.sections():
				for option in parser.options(section):
					if (section, option) != ('global', 'include'):
						value_list = parser.get(section, option).splitlines() # Strip comments
						value_list = map(lambda l: l.rsplit(';', 1)[0].strip(), value_list)
						value_list = filter(lambda l: l != '', value_list)
						container.setEntry(section, option, str.join('\n', value_list), configFile)
		except:
			raise RethrowError('Error while reading configuration file "%s"!' % configFile, ConfigError)
Exemplo n.º 4
0
	def __init__(self):
		# Collect host / user / installation specific config files
		host = socket.gethostbyaddr(socket.gethostname())[0]
		hostCfg = map(lambda c: utils.pathGC('config/%s.conf' % host.split('.', c)[-1]), range(host.count('.') + 1, 0, -1))
		defaultCfg = ['/etc/grid-control.conf', '~/.grid-control.conf', utils.pathGC('config/default.conf')]
		if os.environ.get('GC_CONFIG'):
			defaultCfg.append('$GC_CONFIG')
		fqConfigFiles = map(lambda p: utils.resolvePath(p, mustExist = False), hostCfg + defaultCfg)
		FileConfigFiller.__init__(self, filter(os.path.exists, fqConfigFiles))
Exemplo n.º 5
0
	def _fillContentFromFile(self, configFile, searchPaths, configContent = {}):
		log = logging.getLogger(('config.%s' % utils.getRootName(configFile)).rstrip('.'))
		log.log(logging.INFO1, 'Reading config file %s' % configFile)
		configFile = utils.resolvePath(configFile, searchPaths, ErrorClass = ConfigError)
		configFileData = open(configFile, 'r').readlines()

		# Single pass, non-recursive list retrieval
		tmpConfigContent = {}
		self._fillContentFromSingleFile(configFile, configFileData, searchPaths, tmpConfigContent)
		def getFlatList(section, option):
			for (opt, value, s) in filter(lambda (opt, v, s): opt == option, tmpConfigContent.get(section, [])):
				for entry in utils.parseList(value, None):
					yield entry
Exemplo n.º 6
0
	def __init__(self, filler = None, configFilePath = None):
		def getName(prefix = ''):
			if configFilePath:
				return ('%s.%s' % (prefix, utils.getRootName(configFilePath))).strip('.')
			elif prefix:
				return prefix
			return 'unnamed'

		try:
			pathMain = os.getcwd()
		except Exception:
			raise ConfigError('The current directory does not exist!')
		if configFilePath:
			pathMain = os.path.dirname(utils.resolvePath(configFilePath,
				searchPaths = [os.getcwd()], ErrorClass = ConfigError))

		# Init config containers
		self._curContainer = ConfigContainer('current')
		if filler: # Read in the current configuration ...
			filler.fill(self._curContainer)
		self._curContainer.resolve() # resolve interpolations

		logging.getLogger('config.stored').propagate = False
		oldContainer = ConfigContainer('stored')
		oldContainer.enabled = False

		# Create config view and temporary config interface
		self._view = SimpleConfigView(getName(), oldContainer, self._curContainer)
		self._view.pathDict['search_paths'] = UniqueList([os.getcwd(), pathMain])

		# Determine work directory using config interface with "global" scope
		tmp_config = SimpleConfigInterface(self._view.getView(setSections = ['global']))
		wdBase = tmp_config.getPath('workdir base', pathMain, mustExist = False)
		pathWork = tmp_config.getPath('workdir', os.path.join(wdBase, getName('work')), mustExist = False)
		self._view.pathDict['<WORKDIR>'] = pathWork # tmp_config still has undefinied
		# Set dynamic plugin search path
		sys.path.extend(tmp_config.getPaths('plugin paths', [os.getcwd()]))

		# Determine and load stored config settings
		self._flatCfgPath = os.path.join(pathWork, 'current.conf') # Minimal config file
		self._oldCfgPath = os.path.join(pathWork, 'work.conf') # Config file with saved settings
		if os.path.exists(self._oldCfgPath):
			GeneralFileConfigFiller([self._oldCfgPath]).fill(oldContainer)
			CompatConfigFiller(os.path.join(pathWork, 'task.dat')).fill(oldContainer)
			oldContainer.enabled = True
			oldContainer.setReadOnly()

		# Get persistent variables - only possible after oldContainer was enabled
		self._view.setConfigName(tmp_config.get('config id', getName(), persistent = True))
Exemplo n.º 7
0
	def __init__(self):
		# Collect host / user / installation specific config files
		def resolve_hostname():
			import socket
			host = socket.gethostname()
			try:
				return socket.gethostbyaddr(host)[0]
			except Exception:
				return host
		try:
			host = hang_protection(resolve_hostname, timeout = 5)
			hostCfg = lmap(lambda c: utils.pathPKG('../config/%s.conf' % host.split('.', c)[-1]), irange(host.count('.') + 1, -1, -1))
		except TimeoutException:
			sys.stderr.write('System call to resolve hostname is hanging!\n')
			sys.stderr.flush()
			hostCfg = []
		defaultCfg = ['/etc/grid-control.conf', '~/.grid-control.conf', utils.pathPKG('../config/default.conf')]
		if os.environ.get('GC_CONFIG'):
			defaultCfg.append('$GC_CONFIG')
		log = logging.getLogger('config.default')
		log.log(logging.DEBUG1, 'Possible default config files: %s', str.join(', ', defaultCfg))
		fqConfigFiles = lmap(lambda p: utils.resolvePath(p, mustExist = False), hostCfg + defaultCfg)
		FileConfigFiller.__init__(self, lfilter(os.path.exists, fqConfigFiles), addSearchPath = False)
Exemplo n.º 8
0
	def _getSandboxFiles(self, task, monitor, smList):
		# Prepare all input files
		depList = set(ichain(imap(lambda x: x.getDependencies(), [task] + smList)))
		depPaths = lmap(lambda pkg: utils.pathShare('', pkg = pkg), os.listdir(utils.pathPKG()))
		depFiles = lmap(lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths), depList)
		taskEnv = utils.mergeDicts(imap(lambda x: x.getTaskConfig(), [monitor, task] + smList))
		taskEnv.update({'GC_DEPFILES': str.join(' ', depList), 'GC_USERNAME': self._token.getUsername(),
			'GC_WMS_NAME': self._name})
		taskConfig = sorted(utils.DictFormat(escapeString = True).format(taskEnv, format = 'export %s%s%s\n'))
		varMappingDict = dict(izip(monitor.getTaskConfig().keys(), monitor.getTaskConfig().keys()))
		varMappingDict.update(task.getVarMapping())
		varMapping = sorted(utils.DictFormat(delimeter = ' ').format(varMappingDict, format = '%s%s%s\n'))
		# Resolve wildcards in task input files
		def getTaskFiles():
			for f in task.getSBInFiles():
				matched = glob.glob(f.pathAbs)
				if matched != []:
					for match in matched:
						yield match
				else:
					yield f.pathAbs
		return lchain([monitor.getFiles(), depFiles, getTaskFiles(),
			[VirtualFile('_config.sh', taskConfig), VirtualFile('_varmap.dat', varMapping)]])
Exemplo n.º 9
0
	def _getSandboxFiles(self, module, monitor, smList):
		# Prepare all input files
		depList = set(itertools.chain(*map(lambda x: x.getDependencies(), [module] + smList)))
		depPaths = map(lambda pkg: utils.pathShare('', pkg = pkg), os.listdir(utils.pathGC('packages')))
		depFiles = map(lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths), depList)
		taskEnv = list(itertools.chain(map(lambda x: x.getTaskConfig(), [monitor, module] + smList)))
		taskEnv.append({'GC_DEPFILES': str.join(' ', depList), 'GC_USERNAME': self.proxy.getUsername(),
			'GC_WMS_NAME': self.wmsName})
		taskConfig = sorted(utils.DictFormat(escapeString = True).format(utils.mergeDicts(taskEnv), format = 'export %s%s%s\n'))
		varMappingDict = dict(zip(monitor.getTaskConfig().keys(), monitor.getTaskConfig().keys()))
		varMappingDict.update(module.getVarMapping())
		varMapping = sorted(utils.DictFormat(delimeter = ' ').format(varMappingDict, format = '%s%s%s\n'))
		# Resolve wildcards in module input files
		def getModuleFiles():
			for f in module.getSBInFiles():
				matched = glob.glob(f)
				if matched != []:
					for match in matched:
						yield match
				else:
					yield f
		return list(itertools.chain(monitor.getFiles(), depFiles, getModuleFiles(),
			[utils.VirtualFile('_config.sh', taskConfig), utils.VirtualFile('_varmap.dat', varMapping)]))
Exemplo n.º 10
0
	def resolvePath(self, value, mustExist, errorMsg):
		try:
			return utils.resolvePath(value, self._configView.pathDict.get('search_paths', []), mustExist, ConfigError)
		except Exception:
			raise ConfigError(errorMsg)
Exemplo n.º 11
0
	def resolvePath(self, value, mustExist, errorMsg):
		try:
			return utils.resolvePath(value, [self._pathBase], mustExist, ConfigError)
		except:
			raise RethrowError(errorMsg, ConfigError)