def __getBackendsFromCFG(self, cfgPath): """ Get backends from the configuration and register them in LoggingRoot. This is the new way to get the backends providing a general configuration. :params cfgPath: string of the configuration path """ # We have to put the import line here to avoid a dependancy loop from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC import gConfig # get the second last string representing the component type in the configuration # example : 'Agents', 'Services' component = cfgPath.split("/")[-2] operation = Operations() # Search desired backends in the component desiredBackends = gConfig.getValue("%s/%s" % (cfgPath, 'LogBackends'), []) if not desiredBackends: # Search desired backends in the operation section according to the component type desiredBackends = operation.getValue( "Logging/Default%sBackends" % component, []) if not desiredBackends: # Search desired backends in the operation section desiredBackends = operation.getValue("Logging/DefaultBackends", []) if not desiredBackends: # Default value desiredBackends = ['stdout'] return desiredBackends
def configure( self, csSection, submitPool ): """ Here goes common configuration for all PilotDirectors """ self.configureFromSection( csSection ) self.reloadConfiguration( csSection, submitPool ) # Get the defaults for the Setup where the Director is running opsHelper = Operations() self.installVersion = opsHelper.getValue( cfgPath( 'Pilot', 'Version' ), [ self.installVersion ] )[0] self.installProject = opsHelper.getValue( cfgPath( 'Pilot', 'Project' ), self.installProject ) self.installation = opsHelper.getValue( cfgPath( 'Pilot', 'Installation' ), self.installation ) self.pilotExtensionsList = opsHelper.getValue( "Pilot/Extensions", self.pilotExtensionsList ) self.log.info( '===============================================' ) self.log.info( 'Configuration:' ) self.log.info( '' ) self.log.info( ' Target Grids: ', ', '.join( self.targetGrids ) ) self.log.info( ' Install script: ', self.install ) self.log.info( ' Pilot script: ', self.pilot ) self.log.info( ' Install Ver: ', self.installVersion ) if self.installProject: self.log.info( ' Project: ', self.installProject ) if self.installation: self.log.info( ' Installation: ', self.installation ) if self.extraPilotOptions: self.log.info( ' Extra Options: ', ' '.join( self.extraPilotOptions ) ) self.log.info( ' ListMatch: ', self.enableListMatch ) self.log.info( ' Private %: ', self.privatePilotFraction * 100 ) if self.enableListMatch: self.log.info( ' ListMatch Delay:', self.listMatchDelay ) self.listMatchCache.purgeExpired()
def __getPrefix(self): op = Operations("glast.org") self.userprefix = None res = getProxyInfo() if res["OK"]: if "username" in res["Value"]: user = res["Value"]["username"] self.userprefix = "/glast.org/user/%s/%s/" % (user[0], user) else: self.log.error("Proxy could not be found") return 1 task_category = os.environ["GPL_TASKCATEGORY"] if not task_category: task_category = op.getValue("Pipeline/TaskCategory", None) if not task_category: self.log.error("Could not find task category") return 1 self.prefixDest = op.getValue("Pipeline/StorageElementBasePath", self.userprefix) self.stagingDest = ( self.prefixDest + "/" + task_category + "/" + os.environ["PIPELINE_TASK"] + "/" + os.environ["PIPELINE_STREAM"] ) return 0
def configure(self, csSection, submitPool): """ Here goes common configuration for all PilotDirectors """ self.configureFromSection(csSection) self.reloadConfiguration(csSection, submitPool) # Get the defaults for the Setup where the Director is running opsHelper = Operations() self.installVersion = opsHelper.getValue(cfgPath("Pilot", "Version"), [self.installVersion])[0] self.installProject = opsHelper.getValue(cfgPath("Pilot", "Project"), self.installProject) self.installation = opsHelper.getValue(cfgPath("Pilot", "Installation"), self.installation) self.pilotExtensionsList = opsHelper.getValue("Pilot/Extensions", self.pilotExtensionsList) self.log.info("===============================================") self.log.info("Configuration:") self.log.info("") self.log.info(" Target Grids: ", ", ".join(self.targetGrids)) self.log.info(" Install script: ", self.install) self.log.info(" Pilot script: ", self.pilot) self.log.info(" Pilot modules", self.extraModules) self.log.info(" Install Ver: ", self.installVersion) if self.installProject: self.log.info(" Project: ", self.installProject) if self.installation: self.log.info(" Installation: ", self.installation) if self.extraPilotOptions: self.log.info(" Extra Options: ", " ".join(self.extraPilotOptions)) self.log.info(" ListMatch: ", self.enableListMatch) self.log.info(" Private %: ", self.privatePilotFraction * 100) if self.enableListMatch: self.log.info(" ListMatch Delay:", self.listMatchDelay) self.listMatchCache.purgeExpired()
def configure( self, csSection, submitPool ): """ Here goes common configuration for all PilotDirectors """ self.configureFromSection( csSection ) self.reloadConfiguration( csSection, submitPool ) # Get the defaults for the Setup where the Director is running opsHelper = Operations() self.installVersion = opsHelper.getValue( cfgPath( 'Pilot', 'Version' ), [ self.installVersion ] )[0] self.installProject = opsHelper.getValue( cfgPath( 'Pilot', 'Project' ), self.installProject ) self.installation = opsHelper.getValue( cfgPath( 'Pilot', 'Installation' ), self.installation ) self.pilotExtensionsList = opsHelper.getValue( "Pilot/Extensions", self.pilotExtensionsList ) self.log.info( '===============================================' ) self.log.info( 'Configuration:' ) self.log.info( '' ) self.log.info( ' Target Grids: ', ', '.join( self.targetGrids ) ) self.log.info( ' Install script: ', self.install ) self.log.info( ' Pilot script: ', self.pilot ) self.log.info( ' Pilot modules', self.extraModules ) self.log.info( ' Install Ver: ', self.installVersion ) if self.installProject: self.log.info( ' Project: ', self.installProject ) if self.installation: self.log.info( ' Installation: ', self.installation ) if self.extraPilotOptions: self.log.info( ' Extra Options: ', ' '.join( self.extraPilotOptions ) ) self.log.info( ' ListMatch: ', self.enableListMatch ) self.log.info( ' Private %: ', self.privatePilotFraction * 100 ) if self.enableListMatch: self.log.info( ' ListMatch Delay:', self.listMatchDelay ) self.listMatchCache.purgeExpired()
def getPilotOutput(cls, pilotRef): if not pilotRef.startswith("vm://"): return S_ERROR("Invalid pilot reference %s" % pilotRef) # Get the VM public IP diracID, nPilot = os.path.basename(pilotRef).split(":") result = cls.virtualMachineDB.getUniqueIDByName(diracID) if not result["OK"]: return result uniqueID = result["Value"] result = cls.virtualMachineDB.getInstanceID(uniqueID) if not result["OK"]: return result instanceID = result["Value"] result = cls.virtualMachineDB.getInstanceParameter("PublicIP", instanceID) if not result["OK"]: return result publicIP = result["Value"] op = Operations() privateKeyFile = op.getValue("/Cloud/PrivateKey", "") diracUser = op.getValue("/Cloud/VMUser", "") ssh_str = "%s@%s" % (diracUser, publicIP) cmd = ["ssh", "-i", privateKeyFile, ssh_str, "cat /etc/joboutputs/vm-pilot.%s.log" % nPilot] inst = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE) output, stderr = inst.communicate() if inst.returncode: return S_ERROR("Failed to get pilot output: %s" % stderr) else: return S_OK(output)
def __getCSOptions(self): """Get agent options from the CS.""" self.enabled = self.am_getOption('EnableFlag', False) self.transformationsToIgnore = self.am_getOption( 'TransformationsToIgnore', []) self.getJobInfoFromJDLOnly = self.am_getOption('JobInfoFromJDLOnly', False) self.transformationStatus = self.am_getOption('TransformationStatus', ['Active', 'Completing']) ops = Operations() extendableTTypes = set( ops.getValue('Transformations/ExtendableTransfTypes', ['MCSimulation'])) dataProcessing = set(ops.getValue('Transformations/DataProcessing', [])) self.transNoInput = self.am_getOption('TransformationsNoInput', list(extendableTTypes)) self.transWithInput = self.am_getOption( 'TransformationsWithInput', list(dataProcessing - extendableTTypes)) self.transformationTypes = self.transWithInput + self.transNoInput self.log.notice('Will treat transformations without input files', self.transNoInput) self.log.notice('Will treat transformations with input files', self.transWithInput) self.addressTo = self.am_getOption('MailTo', []) self.addressFrom = self.am_getOption('MailFrom', '') self.printEveryNJobs = self.am_getOption('PrintEvery', 200)
def initialize(self): """ Flags useESForJobParametersFlag (in /Operations/[]/Services/JobMonitoring/) have bool value (True/False) and determines the switching of backends from MySQL to ElasticSearch for the JobParameters DB table. For version v7r0, the MySQL backend is (still) the default. """ credDict = self.getRemoteCredentials() self.ownerDN = credDict['DN'] self.ownerGroup = credDict['group'] operations = Operations(group=self.ownerGroup) self.globalJobsInfo = operations.getValue( '/Services/JobMonitoring/GlobalJobsInfo', True) self.jobPolicy = JobPolicy(self.ownerDN, self.ownerGroup, self.globalJobsInfo) self.jobPolicy.jobDB = gJobDB useESForJobParametersFlag = operations.getValue( '/Services/JobMonitoring/useESForJobParametersFlag', False) global gElasticJobDB if useESForJobParametersFlag: gElasticJobDB = ElasticJobDB() self.log.verbose("Using ElasticSearch for JobParameters") return S_OK()
def __getBackendsFromCFG(self, cfgPath): """ Get backends from the configuration and register them in LoggingRoot. This is the new way to get the backends providing a general configuration. :params cfgPath: string of the configuration path """ # We have to put the import line here to avoid a dependancy loop from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC import gConfig # get the second last string representing the component type in the configuration # example : 'Agents', 'Services' component = cfgPath.split("/")[-2] operation = Operations() # Search desired backends in the component desiredBackends = gConfig.getValue("%s/%s" % (cfgPath, 'LogBackends'), []) if not desiredBackends: # Search desired backends in the operation section according to the # component type desiredBackends = operation.getValue( "Logging/Default%sBackends" % component, []) if not desiredBackends: # Search desired backends in the operation section desiredBackends = operation.getValue("Logging/DefaultBackends", []) if not desiredBackends: # Default value desiredBackends = ['stdout'] return desiredBackends
def check(self): """ Check that the manifest is OK """ for k in ["OwnerName", "OwnerDN", "OwnerGroup", "DIRACSetup"]: if k not in self.__manifest: return S_ERROR("Missing var %s in manifest" % k) # Check CPUTime result = self.__checkNumericalVar("CPUTime", 86400, 100, 500000) if not result["OK"]: return result result = self.__checkNumericalVar("Priority", 1, 0, 10) if not result["OK"]: return result maxInputData = Operations().getValue("JobDescription/MaxInputData", 500) result = self.__checkMaxInputData(maxInputData) if not result["OK"]: return result operation = Operations(group=self.__manifest["OwnerGroup"]) allowedJobTypes = operation.getValue("JobDescription/AllowedJobTypes", ["User", "Test", "Hospital"]) transformationTypes = operation.getValue( "Transformations/DataProcessing", []) result = self.__checkMultiChoice("JobType", allowedJobTypes + transformationTypes) if not result["OK"]: return result return S_OK()
def getTarBallLocation(app, config, dummy_area): """ Get the tar ball location. """ ops = Operations() appName = app[0] appVersion = app[1] appName = appName.lower() app_tar = ops.getValue( '/AvailableTarBalls/%s/%s/%s/TarBall' % (config, appName, appVersion), '') overwrite = ops.getValue( '/AvailableTarBalls/%s/%s/%s/Overwrite' % (config, appName, appVersion), False) md5sum = ops.getValue( '/AvailableTarBalls/%s/%s/%s/Md5Sum' % (config, appName, appVersion), '') gLogger.info("Looking for application %s%s for config %s:" % (appName, appVersion, config)) if not app_tar: gLogger.error('Could not find tar ball for %s %s' % (appName, appVersion)) return S_ERROR('Could not find tar ball for %s %s' % (appName, appVersion)) tarballURL = ops.getValue( '/AvailableTarBalls/%s/%s/TarBallURL' % (config, appName), '') if not tarballURL: gLogger.error('Could not find tarballURL in CS for %s %s' % (appName, appVersion)) return S_ERROR('Could not find tarballURL in CS') return S_OK([app_tar, tarballURL, overwrite, md5sum])
def getPilotOutput(pilotRef): if not pilotRef.startswith('vm://'): return S_ERROR('Invalid pilot reference %s' % pilotRef) # Get the VM public IP diracID, nPilot = os.path.basename(pilotRef).split(':') result = gVirtualMachineDB.getUniqueIDByName(diracID) if not result['OK']: return result uniqueID = result['Value'] result = gVirtualMachineDB.getInstanceID(uniqueID) if not result['OK']: return result instanceID = result['Value'] result = gVirtualMachineDB.getInstanceParameter("PublicIP", instanceID) if not result['OK']: return result publicIP = result['Value'] op = Operations() privateKeyFile = op.getValue('/Cloud/PrivateKey', '') diracUser = op.getValue('/Cloud/VMUser', '') cmd = 'ssh -i %s %s@%s "cat /etc/joboutputs/vm-pilot.%s.log"' % ( privateKeyFile, diracUser, publicIP, nPilot) status, output = commands.getstatusoutput(cmd) if status: return S_ERROR('Failed to get pilot output: %s' % output) else: return S_OK(output)
def registerSwitchesAndParseCommandLine(self): """Register the default plus additional parameters and parse options. :param list options: list of three tuple for options to add to the script :param list flags: list of three tuple for flags to add to the script :param str opName """ for short, longOption, doc in self.options: Script.registerSwitch(short + ':' if short else '', longOption + '=', doc) for short, longOption, doc in self.flags: Script.registerSwitch(short, longOption, doc) self.switches[longOption] = False Script.parseCommandLine() if Script.getPositionalArgs(): Script.showHelp() DIRAC.exit(1) ops = Operations() if not ops.getValue('DataManagement/ArchiveFiles/Enabled', False): sLog.error( 'The "ArchiveFiles" operation is not enabled, contact your administrator!' ) DIRAC.exit(1) for _short, longOption, _doc in self.options: defaultValue = ops.getValue( 'DataManagement/ArchiveFiles/%s' % longOption, None) if defaultValue: sLog.verbose( 'Found default value in the CS for %r with value %r' % (longOption, defaultValue)) self.switches[longOption] = defaultValue for _short, longOption, _doc in self.flags: defaultValue = ops.getValue( 'DataManagement/ArchiveFiles/%s' % longOption, False) if defaultValue: sLog.verbose( 'Found default value in the CS for %r with value %r' % (longOption, defaultValue)) self.switches[longOption] = defaultValue for switch in Script.getUnprocessedSwitches(): for short, longOption, doc in self.options: if switch[0] == short or switch[0].lower() == longOption.lower( ): sLog.verbose('Found switch %r with value %r' % (longOption, switch[1])) self.switches[longOption] = switch[1] break for short, longOption, doc in self.flags: if switch[0] == short or switch[0].lower() == longOption.lower( ): self.switches[longOption] = True break self.checkSwitches() self.switches['DryRun'] = not self.switches.get('Execute', False) self.switches['SourceSE'] = self.switches.get('SourceSE', '').split(',')
class RssConfiguration: ''' RssConfiguration: { Config: { State : Active | InActive, Cache : 300, FromAddress : '*****@*****.**' StatusType : { default : all, StorageElement: ReadAccess, WriteAccess, CheckAccess, RemoveAccess } } } ''' def __init__(self): self.opsHelper = Operations() def getConfigCache(self, default=300): ''' Gets from <pathToRSSConfiguration>/Config the value of Cache ''' return self.opsHelper.getValue('%s/Config/Cache' % _rssConfigPath, default) def getConfigFromAddress(self, default=None): ''' Gets from <pathToRSSConfiguration>/Config the value of FromAddress ''' return self.opsHelper.getValue( '%s/Config/FromAddress' % _rssConfigPath, default) def getConfigStatusType(self, elementType=None): ''' Gets all the status types per elementType, if not given, it takes default from CS. If not, hardcoded variable DEFAULT. ''' _DEFAULTS = ('all', ) res = self.opsHelper.getOptionsDict('%s/Config/StatusTypes' % _rssConfigPath) if res['OK']: if elementType in res['Value']: return List.fromChar(res['Value'][elementType]) if 'default' in res['Value']: return List.fromChar(res['Value']['default']) return _DEFAULTS
class RssConfiguration: ''' RssConfiguration: { Config: { State : Active | InActive, Cache : 300, FromAddress : '*****@*****.**' StatusType : { default : all, StorageElement: ReadAccess, WriteAccess, CheckAccess, RemoveAccess } } } ''' def __init__( self ): self.opsHelper = Operations() def getConfigCache( self, default = 300 ): ''' Gets from <pathToRSSConfiguration>/Config the value of Cache ''' return self.opsHelper.getValue( '%s/Config/Cache' % _rssConfigPath, default ) def getConfigFromAddress( self, default = None ): ''' Gets from <pathToRSSConfiguration>/Config the value of FromAddress ''' return self.opsHelper.getValue( '%s/Config/FromAddress' % _rssConfigPath, default ) def getConfigStatusType( self, elementType = None ): ''' Gets all the status types per elementType, if not given, it takes default from CS. If not, hardcoded variable DEFAULT. ''' _DEFAULTS = ( 'all', ) res = self.opsHelper.getOptionsDict( '%s/Config/StatusTypes' % _rssConfigPath ) if res[ 'OK' ]: if elementType in res[ 'Value' ]: return List.fromChar( res[ 'Value' ][ elementType ] ) if 'default' in res[ 'Value' ]: return List.fromChar( res[ 'Value' ][ 'default' ] ) return _DEFAULTS
def getShifterProxy(shifterType, fileName=False): """ This method returns a shifter's proxy :param shifterType: ProductionManager / DataManager... """ if fileName: try: os.makedirs(os.path.dirname(fileName)) except OSError: pass opsHelper = Operations() userName = opsHelper.getValue(cfgPath('Shifter', shifterType, 'User'), '') if not userName: return S_ERROR("No shifter User defined for %s" % shifterType) result = CS.getDNForUsername(userName) if not result['OK']: return result userDN = result['Value'][0] result = CS.findDefaultGroupForDN(userDN) if not result['OK']: return result defaultGroup = result['Value'] userGroup = opsHelper.getValue(cfgPath('Shifter', shifterType, 'Group'), defaultGroup) vomsAttr = CS.getVOMSAttributeForGroup(userGroup) if vomsAttr: gLogger.info("Getting VOMS [%s] proxy for shifter %s@%s (%s)" % (vomsAttr, userName, userGroup, userDN)) result = gProxyManager.downloadVOMSProxyToFile(userDN, userGroup, filePath=fileName, requiredTimeLeft=86400, cacheTime=86400) else: gLogger.info("Getting proxy for shifter %s@%s (%s)" % (userName, userGroup, userDN)) result = gProxyManager.downloadProxyToFile(userDN, userGroup, filePath=fileName, requiredTimeLeft=86400, cacheTime=86400) if not result['OK']: return result chain = result['chain'] fileName = result['Value'] return S_OK({ 'DN': userDN, 'username': userName, 'group': userGroup, 'chain': chain, 'proxyFile': fileName })
def sync(self): """ Main synchronizer method. """ ops = Operations() self._checksumDict = {} self.pilotFileServer = ops.getValue("Pilot/pilotFileServer", self.pilotFileServer) if not self.pilotFileServer: self.log.fatal( "The /Operations/<Setup>/Pilot/pilotFileServer option is not defined" ) self.log.fatal( "Pilot 3 files won't be updated, and you won't be able to send pilots" ) return S_OK( "The /Operations/<Setup>/Pilot/pilotFileServer option is not defined" ) self.workDir = ops.getValue("Pilot/workDir", self.workDir) if self.workDir: try: os.mkdir(self.workDir) except OSError: pass self.log.notice( '-- Synchronizing the content of the JSON file with the content of the CS --', '(%s)' % self.jsonFile) self.pilotRepo = ops.getValue("Pilot/pilotRepo", self.pilotRepo) self.pilotVORepo = ops.getValue("Pilot/pilotVORepo", self.pilotVORepo) self.projectDir = ops.getValue("Pilot/projectDir", self.projectDir) self.pilotScriptPath = ops.getValue("Pilot/pilotScriptsPath", self.pilotScriptPath) self.pilotVOScriptPath = ops.getValue("Pilot/pilotVOScriptsPath", self.pilotVOScriptPath) self.pilotRepoBranch = ops.getValue("Pilot/pilotRepoBranch", self.pilotRepoBranch) self.pilotVORepoBranch = ops.getValue("Pilot/pilotVORepoBranch", self.pilotVORepoBranch) self.uploadToWebApp = ops.getValue("Pilot/uploadToWebApp", True) res = self._syncJSONFile() if not res['OK']: return res self.log.notice( '-- Synchronizing the pilot scripts with the content of the repository --', '(%s)' % self.pilotRepo) self._syncScripts() self._syncChecksum() return S_OK()
def getShifterProxy(shifterType, fileName=False): """This method returns a shifter's proxy :param str shifterType: ProductionManager / DataManager... :param str fileName: file name :return: S_OK(dict)/S_ERROR() """ if fileName: mkDir(os.path.dirname(fileName)) opsHelper = Operations() userName = opsHelper.getValue(cfgPath("Shifter", shifterType, "User"), "") if not userName: return S_ERROR("No shifter User defined for %s" % shifterType) result = Registry.getDNForUsername(userName) if not result["OK"]: return result userDN = result["Value"][0] result = Registry.findDefaultGroupForDN(userDN) if not result["OK"]: return result defaultGroup = result["Value"] userGroup = opsHelper.getValue(cfgPath("Shifter", shifterType, "Group"), defaultGroup) vomsAttr = Registry.getVOMSAttributeForGroup(userGroup) if vomsAttr: gLogger.info("Getting VOMS [%s] proxy for shifter %s@%s (%s)" % (vomsAttr, userName, userGroup, userDN)) result = gProxyManager.downloadVOMSProxyToFile(userDN, userGroup, filePath=fileName, requiredTimeLeft=86400, cacheTime=86400) else: gLogger.info("Getting proxy for shifter %s@%s (%s)" % (userName, userGroup, userDN)) result = gProxyManager.downloadProxyToFile(userDN, userGroup, filePath=fileName, requiredTimeLeft=86400, cacheTime=86400) if not result["OK"]: return result chain = result["chain"] fileName = result["Value"] return S_OK({ "DN": userDN, "username": userName, "group": userGroup, "chain": chain, "proxyFile": fileName })
def __init__(self, argumentsDict): """ Standard constructor """ self.arguments = argumentsDict self.name = COMPONENT_NAME self.log = gLogger.getSubLogger(self.name) op = Operations() self.arguments.setdefault('Configuration', {})['AllReplicas'] = op.getValue('InputDataPolicy/AllReplicas', False) self.arguments['Configuration'].setdefault('Protocol', op.getValue('InputDataPolicy/Protocols/Local', [])) self.arguments['Configuration'].setdefault('RemoteProtocol', op.getValue('InputDataPolicy/Protocols/Remote', [])) # By default put input data into the current directory self.arguments.setdefault('InputDataDirectory', 'CWD')
def sync(self): """ Main synchronizer method. """ ops = Operations() self.pilotFileServer = ops.getValue("Pilot/pilotFileServer", self.pilotFileServer) if not self.pilotFileServer: self.log.fatal("The /Operations/<Setup>/Pilot/pilotFileServer option is not defined") self.log.fatal("Pilot 3 files won't be updated, and you won't be able to send pilots") return S_OK("The /Operations/<Setup>/Pilot/pilotFileServer option is not defined") self.log.notice('-- Synchronizing the content of the JSON file with the content of the CS --', '(%s)' % self.jsonFile) self.pilotRepo = ops.getValue("Pilot/pilotRepo", self.pilotRepo) self.pilotVORepo = ops.getValue("Pilot/pilotVORepo", self.pilotVORepo) self.projectDir = ops.getValue("Pilot/projectDir", self.projectDir) self.pilotScriptPath = ops.getValue("Pilot/pilotScriptsPath", self.pilotScriptPath) self.pilotVOScriptPath = ops.getValue("Pilot/pilotVOScriptsPath", self.pilotVOScriptPath) self.pilotRepoBranch = ops.getValue("Pilot/pilotRepoBranch", self.pilotRepoBranch) self.pilotVORepoBranch = ops.getValue("Pilot/pilotVORepoBranch", self.pilotVORepoBranch) self._syncJSONFile() self.log.notice('-- Synchronizing the pilot scripts with the content of the repository --', '(%s)' % self.pilotRepo) self._syncScripts() return S_OK()
def getShifterProxy( shifterType, fileName = False ): """ This method returns a shifter's proxy :param shifterType: ProductionManager / DataManager... """ if fileName: try: os.makedirs( os.path.dirname( fileName ) ) except OSError: pass opsHelper = Operations() userName = opsHelper.getValue( cfgPath( 'Shifter', shifterType, 'User' ), '' ) if not userName: return S_ERROR( "No shifter User defined for %s" % shifterType ) result = CS.getDNForUsername( userName ) if not result[ 'OK' ]: return result userDN = result[ 'Value' ][0] result = CS.findDefaultGroupForDN( userDN ) if not result['OK']: return result defaultGroup = result['Value'] userGroup = opsHelper.getValue( cfgPath( 'Shifter', shifterType, 'Group' ), defaultGroup ) vomsAttr = CS.getVOMSAttributeForGroup( userGroup ) if vomsAttr: gLogger.info( "Getting VOMS [%s] proxy for shifter %s@%s (%s)" % ( vomsAttr, userName, userGroup, userDN ) ) result = gProxyManager.downloadVOMSProxyToFile( userDN, userGroup, filePath = fileName, requiredTimeLeft = 86400, cacheTime = 86400 ) else: gLogger.info( "Getting proxy for shifter %s@%s (%s)" % ( userName, userGroup, userDN ) ) result = gProxyManager.downloadProxyToFile( userDN, userGroup, filePath = fileName, requiredTimeLeft = 86400, cacheTime = 86400 ) if not result[ 'OK' ]: return result chain = result[ 'chain' ] fileName = result[ 'Value' ] return S_OK( { 'DN' : userDN, 'username' : userName, 'group' : userGroup, 'chain' : chain, 'proxyFile' : fileName } )
def initialize(self): """ Flags gESFlag and gMySQLFlag have bool values (True/False) derived from dirac.cfg configuration file Determines the switching of ElasticSearch and MySQL backends """ global gElasticJobDB, gJobDB gESFlag = self.srv_getCSOption('useES', False) if gESFlag: gElasticJobDB = ElasticJobDB() gMySQLFlag = self.srv_getCSOption('useMySQL', True) if not gMySQLFlag: gJobDB = False credDict = self.getRemoteCredentials() self.ownerDN = credDict['DN'] self.ownerGroup = credDict['group'] operations = Operations(group=self.ownerGroup) self.globalJobsInfo = operations.getValue('/Services/JobMonitoring/GlobalJobsInfo', True) self.jobPolicy = JobPolicy(self.ownerDN, self.ownerGroup, self.globalJobsInfo) self.jobPolicy.jobDB = gJobDB return S_OK()
def getValidStatusTypes(): ''' Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/Resources ''' DEFAULTS = { 'Site' : { 'StatusType' : "''" }, 'Service' : { 'StatusType' : "''" }, 'Resource' : { 'StatusType' : "''" }, 'StorageElement': { 'StatusType' : [ 'Read', 'Write', 'Remove', 'Check' ] } } opHelper = Operations() sections = opHelper.getSections( 'RSSConfiguration/GeneralConfig/Resources' ) if not sections[ 'OK' ]: return DEFAULTS result = {} for section in sections[ 'Value' ]: res = opHelper.getValue( 'RSSConfiguration/GeneralConfig/Resources/%s/StatusType' % section ) if res is None: if DEFAULTS.has_key( section ): result[ section ] = { 'StatusType' : DEFAULTS[ section ] } else: result[ section ] = { 'StatusType' : None } else: result[ section ] = { 'StatusType' : Utils.getTypedList( res ) } return result
def _getSEList( self, SEType = 'ProductionOutputs', DataType = 'SimtelProd' ): """ get from CS the list of available SE for data upload """ opsHelper = Operations() optionName = os.path.join( SEType, DataType ) SEList = opsHelper.getValue( optionName , [] ) SEList = List.randomize( SEList ) DIRAC.gLogger.notice( 'List of %s SE: %s ' % ( SEType, SEList ) ) # # Check if the local SE is in the list. If yes try it first by reversing list order localSEList = [] res = getSEsForSite( DIRAC.siteName() ) if res['OK']: localSEList = res['Value'] retainedlocalSEList = [] for localSE in localSEList: if localSE in SEList: DIRAC.gLogger.notice( 'The local Storage Element is an available SE: ', localSE ) retainedlocalSEList.append( localSE ) SEList.remove( localSE ) SEList = retainedlocalSEList + SEList if len( SEList ) == 0: return DIRAC.S_ERROR( 'Error in building SEList' ) return DIRAC.S_OK( SEList )
def resolveDeps(sysconfig, appli, appversion): """ Resolve the dependencies :param str sysconfig: system configuration :param str appli: application name :param str appversion: application version :return: list of dictionaries """ log = gLogger.getSubLogger("resolveDeps") ops = Operations() deps = ops.getSections('/AvailableTarBalls/%s/%s/%s/Dependencies' % (sysconfig, appli, appversion), '') depsarray = [] if deps['OK']: for dep in deps['Value']: vers = ops.getValue('/AvailableTarBalls/%s/%s/%s/Dependencies/%s/version' % (sysconfig, appli, appversion, dep), '') depvers = '' if vers: depvers = vers else: log.error("Retrieving dependency version for %s failed, skipping to next !" % (dep)) continue log.verbose("Found dependency %s %s" % (dep, depvers)) depdict = {} depdict["app"] = dep depdict["version"] = depvers depsarray.append(depdict) ##resolve recursive dependencies depsofdeps = resolveDeps(sysconfig, dep, depvers) depsarray.extend(depsofdeps) else: log.verbose("Could not find any dependency for %s %s, ignoring" % (appli, appversion)) return depsarray
def submitProbeJobs(self, ce): """ Submit some jobs to the CEs """ #need credentials, should be there since the initialize from DIRAC.Interfaces.API.Dirac import Dirac d = Dirac() from DIRAC.Interfaces.API.Job import Job from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations import os ops = Operations("glast.org") scriptname = ops.getValue("ResourceStatus/SofwareManagementScript", self.script) j = Job() j.setDestinationCE(ce) j.setCPUTime(1000) j.setName("Probe %s" % ce) j.setJobGroup("SoftwareProbe") j.setExecutable("%s/GlastDIRAC/ResourceStatusSystem/Client/%s" % (os.environ['DIRAC'], scriptname), logFile='SoftwareProbe.log') j.setOutputSandbox('*.log') res = d.submit(j) if not res['OK']: return res return S_OK()
def submitProbeJobs(self, ce): """ Submit some jobs to the CEs """ #need credentials, should be there since the initialize from DIRAC.Interfaces.API.Dirac import Dirac d = Dirac() from DIRAC.Interfaces.API.Job import Job from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations import DIRAC ops = Operations() scriptname = ops.getValue("ResourceStatus/SofwareManagementScript", self.script) j = Job() j.setDestinationCE(ce) j.setCPUTime(1000) j.setName("Probe %s" % ce) j.setJobGroup("SoftwareProbe") j.setExecutable("%s/GlastDIRAC/ResourceStatusSystem/Client/%s" % (DIRAC.rootPath, scriptname), logFile='SoftwareProbe.log') j.setOutputSandbox('*.log') res = d.submit(j) if not res['OK']: return res return S_OK()
def initializeLesHouchesFileManagerHandler(serviceInfo): """ Initialize the service """ ops = Operations() res = ops.getOptionsDict("/Models") if not res["OK"]: return res templates = res["Value"] cfgPath = serviceInfo["serviceSectionPath"] location = "" location = ops.getValue("%s/BasePath" % cfgPath, location) if not location: gLogger.error("Path to LesHouches files not defined") return S_ERROR("Path to LesHouches files not defined in CS") missing = False global ModelsDict for template, tfile in templates.items(): ModelsDict[template] = {} ModelsDict[template]["file"] = tfile if not tfile: ModelsDict[template]["content"] = [""] continue file_path = os.path.join([location, tfile]) if not os.path.exists(file_path): gLogger.error("Missing %s" % file_path) missing = True break LesHouchesFile = open(file_path, "r") ModelsDict[template]["content"] = LesHouchesFile.readlines() LesHouchesFile.close() if missing: return S_ERROR("File missing") return S_OK()
def __init__( self, **kwargs ): Client.__init__( self, **kwargs ) opsH = Operations() self.maxResetCounter = opsH.getValue( 'Productions/ProductionFilesMaxResetCounter', 10 ) self.setServer( 'Transformation/TransformationManager' )
def __init__(self, argumentsDict): """Standard constructor""" self.arguments = argumentsDict self.name = COMPONENT_NAME self.log = gLogger.getSubLogger(self.name) op = Operations() self.arguments.setdefault("Configuration", {})["AllReplicas"] = op.getValue( "InputDataPolicy/AllReplicas", False) self.arguments["Configuration"].setdefault( "Protocol", op.getValue("InputDataPolicy/Protocols/Local", [])) self.arguments["Configuration"].setdefault( "RemoteProtocol", op.getValue("InputDataPolicy/Protocols/Remote", [])) # By default put input data into the current directory self.arguments.setdefault("InputDataDirectory", "CWD")
def __getCSOptions(self): """Get agent options from the CS.""" self.enabled = self.am_getOption("EnableFlag", False) self.transformationsToIgnore = self.am_getOption("TransformationsToIgnore", []) self.getJobInfoFromJDLOnly = self.am_getOption("JobInfoFromJDLOnly", False) self.transformationStatus = self.am_getOption("TransformationStatus", ["Active", "Completing"]) ops = Operations() extendableTTypes = set(ops.getValue("Transformations/ExtendableTransfTypes", ["MCSimulation"])) dataProcessing = set(ops.getValue("Transformations/DataProcessing", [])) self.transNoInput = self.am_getOption("TransformationsNoInput", list(extendableTTypes)) self.transWithInput = self.am_getOption("TransformationsWithInput", list(dataProcessing - extendableTTypes)) self.transformationTypes = self.transWithInput + self.transNoInput self.log.notice("Will treat transformations without input files", self.transNoInput) self.log.notice("Will treat transformations with input files", self.transWithInput) self.addressTo = self.am_getOption("MailTo", []) self.addressFrom = self.am_getOption("MailFrom", "") self.printEveryNJobs = self.am_getOption("PrintEvery", 200)
def __init__(self, **kwargs): Client.__init__(self, **kwargs) opsH = Operations() self.maxResetCounter = opsH.getValue( 'Productions/ProductionFilesMaxResetCounter', 10) self.setServer('Transformation/TransformationManager')
def __getOwnerGroupDN(self, shifterType): opsHelper = Operations() userName = opsHelper.getValue( cfgPath('BoincShifter', shifterType, 'User'), '') if not userName: return S_ERROR("No shifter User defined for %s" % shifterType) result = CS.getDNForUsername(userName) if not result['OK']: return result userDN = result['Value'][0] result = CS.findDefaultGroupForDN(userDN) if not result['OK']: return result defaultGroup = result['Value'] userGroup = opsHelper.getValue( cfgPath('BoincShifter', shifterType, 'Group'), defaultGroup) return userDN, userGroup, userName
def __init__(self): """c'tor Just setting defaults """ self.workDir = "" # Working directory where the files are going to be stored # domain name of the web server(s) used to upload the pilot json file and the pilot scripts self.pilotFileServer = "" # pilot sync default parameters self.pilotRepo = "https://github.com/DIRACGrid/Pilot.git" # repository of the pilot self.pilotVORepo = "" # repository of the VO that can contain a pilot extension self.pilotSetup = gConfig.getValue("/DIRAC/Setup", "") self.projectDir = "" # where the find the pilot scripts in the VO pilot repository self.pilotScriptPath = "Pilot" # where the find the pilot scripts in the pilot repository self.pilotVOScriptPath = "" self.pilotRepoBranch = "master" self.pilotVORepoBranch = "master" self.log = gLogger.getSubLogger(__name__) ops = Operations() # Overriding parameters from the CS self.pilotRepo = ops.getValue("Pilot/pilotRepo", self.pilotRepo) self.pilotVORepo = ops.getValue("Pilot/pilotVORepo", self.pilotVORepo) self.projectDir = ops.getValue("Pilot/projectDir", self.projectDir) self.pilotScriptPath = ops.getValue("Pilot/pilotScriptsPath", self.pilotScriptPath) self.pilotVOScriptPath = ops.getValue("Pilot/pilotVOScriptsPath", self.pilotVOScriptPath) self.pilotRepoBranch = ops.getValue("Pilot/pilotRepoBranch", self.pilotRepoBranch) self.pilotVORepoBranch = ops.getValue("Pilot/pilotVORepoBranch", self.pilotVORepoBranch)
def __init__(self, argumentsDict): """ Standard constructor """ self.arguments = argumentsDict self.name = COMPONENT_NAME self.log = gLogger.getSubLogger(self.name) op = Operations() self.arguments.setdefault('Configuration', {})['AllReplicas'] = op.getValue( 'InputDataPolicy/AllReplicas', False) self.arguments['Configuration'].setdefault( 'Protocol', op.getValue('InputDataPolicy/Protocols/Local', [])) self.arguments['Configuration'].setdefault( 'RemoteProtocol', op.getValue('InputDataPolicy/Protocols/Remote', [])) # By default put input data into the current directory self.arguments.setdefault('InputDataDirectory', 'CWD')
def __init__(self, **kwargs): """Simple constructor""" super(TransformationClient, self).__init__(**kwargs) opsH = Operations() self.maxResetCounter = opsH.getValue( "Transformations/FilesMaxResetCounter", 10) self.setServer("Transformation/TransformationManager")
def allowedBkg( bkg, energy = None, detector = None, detectormodel = None, machine = 'clic_cdr' ): """ Check is supplied bkg is allowed """ #gLogger.info("Those are the arguments: %s, %s, %s, %s, %s" % (bkg, energy, detector, detectormodel, machine) ) ops = Operations() #bkg_allowed = [ 'gghad', 'pairs' ] #if not bkg in bkg_allowed: # return S_ERROR( "Bkg not allowed" ) res = -1 if energy: if detectormodel: res = ops.getValue( "/Overlay/%s/%s/%s/%s/ProdID" % (machine, energy, detectormodel, bkg), -1 ) if res < 0: return S_ERROR( "No background to overlay" ) if detector:##needed for backward compatibility res = ops.getValue( "/Overlay/%s/%s/%s/%s/ProdID" % (machine, detector, energy, bkg), -1 ) if res < 0 : return S_ERROR( "No background to overlay" ) return S_OK(res)
def allowedBkg( bkg, energy = None, detector = None, detectormodel = None, machine = 'clic_cdr' ): """ Check is supplied bkg is allowed """ #gLogger.info("Those are the arguments: %s, %s, %s, %s, %s" % (bkg, energy, detector, detectormodel, machine) ) ops = Operations() #bkg_allowed = [ 'gghad', 'pairs' ] #if not bkg in bkg_allowed: # return S_ERROR( "Bkg not allowed" ) res = -1 if energy: if detectormodel: res = ops.getValue( "/Overlay/%s/%s/%s/%s/ProdID" % (machine, energy, detectormodel, bkg), -1 ) if res < 0: return S_ERROR( "No background to overlay" ) if detector:##needed for backward compatibility res = ops.getValue( "/Overlay/%s/%s/%s/%s/ProdID" % (machine, detector, energy, bkg), -1 ) if res < 0 : return S_ERROR( "No background to overlay" ) return S_OK(res)
def __init__(self, **kwargs): """ Simple constructor """ Client.__init__(self, **kwargs) opsH = Operations() self.maxResetCounter = opsH.getValue( 'Transformations/FilesMaxResetCounter', 10) self.setServer('Transformation/TransformationManager')
def getCountryMappingTier1( country ): """ Returns the Tier1 site mapped to a country code """ opsHelper = Operations() res = getCountryMapping( country ) if not res['OK']: return res mappedCountry = res['Value'] tier1 = opsHelper.getValue( '/Countries/%s/Tier1' % mappedCountry, '' ) if not tier1: return S_ERROR( "No Tier1 assigned to %s" % mappedCountry ) return S_OK( tier1 )
def getCountryMappingTier1(country): """ Returns the Tier1 site mapped to a country code """ opsHelper = Operations() res = getCountryMapping(country) if not res['OK']: return res mappedCountry = res['Value'] tier1 = opsHelper.getValue('/Countries/%s/Tier1' % mappedCountry, '') if not tier1: return S_ERROR("No Tier1 assigned to %s" % mappedCountry) return S_OK(tier1)
def constructUserLFNs(jobID, vo, owner, outputFiles, outputPath): """ This method is used to supplant the standard job wrapper output data policy for ILC. The initial convention adopted for user output files is the following: If outputpath is not defined: <vo>/user/<initial e.g. s>/<owner e.g. sposs>/<yearMonth e.g. 2010_02>/<subdir>/<fileName> Otherwise: <vo>/user/<initial e.g. s>/<owner e.g. sposs>/<outputPath>/<fileName> """ initial = owner[:1] subdir = str(jobID/1000) timeTup = datetime.date.today().timetuple() yearMonth = '%s_%s' % (timeTup[0], string.zfill(str(timeTup[1]), 2)) outputLFNs = {} if not vo: #res = gConfig.getOption("/DIRAC/VirtualOrganization", "ilc") res = getVOfromProxyGroup() if not res['OK']: gLogger.error('Could not get VO from CS, assuming ilc') vo = 'ilc' else: vo = res['Value'] ops = Operations(vo = vo) lfn_prefix = ops.getValue("LFNUserPrefix", "user") #Strip out any leading or trailing slashes but allow fine structure if outputPath: outputPathList = string.split(outputPath, os.sep) newPath = [] for i in outputPathList: if i: newPath.append(i) outputPath = string.join(newPath, os.sep) if not type(outputFiles) == types.ListType: outputFiles = [outputFiles] for outputFile in outputFiles: #strip out any fine structure in the output file specified by the user, restrict to output file names #the output path field can be used to describe this outputFile = outputFile.replace('LFN:', '') lfn = '' if outputPath: lfn = os.sep+os.path.join(vo, lfn_prefix, initial, owner, outputPath + os.sep + os.path.basename(outputFile)) else: lfn = os.sep+os.path.join(vo, lfn_prefix, initial, owner, yearMonth, subdir, str(jobID)) + os.sep + os.path.basename(outputFile) outputLFNs[outputFile] = lfn outputData = outputLFNs.values() if outputData: gLogger.info('Created the following output data LFN(s):\n%s' % (string.join(outputData, '\n'))) else: gLogger.info('No output LFN(s) constructed') return S_OK(outputData)
def getSteeringFileDirName(platform, application, applicationVersion): """ Locate the path of the steering file directory assigned to the specified application """ ops = Operations() version = ops.getValue('/AvailableTarBalls/%s/%s/%s/Dependencies/steeringfiles/version' % (platform, application, applicationVersion), '') if not version: return S_ERROR("Could not find attached SteeringFile version") return getSteeringFileDir(platform, version)
def initialize( self ): credDict = self.getRemoteCredentials() self.ownerDN = credDict['DN'] self.ownerGroup = credDict['group'] operations = Operations( group = self.ownerGroup ) self.globalJobsInfo = operations.getValue( '/Services/JobMonitoring/GlobalJobsInfo', True ) self.jobPolicy = JobPolicy( self.ownerDN, self.ownerGroup, self.globalJobsInfo ) self.jobPolicy.setJobDB( gJobDB ) return S_OK()
def initialize(self): credDict = self.getRemoteCredentials() self.ownerDN = credDict['DN'] self.ownerGroup = credDict['group'] operations = Operations(group=self.ownerGroup) self.globalJobsInfo = operations.getValue('/Services/JobMonitoring/GlobalJobsInfo', True) self.jobPolicy = JobPolicy(self.ownerDN, self.ownerGroup, self.globalJobsInfo) self.jobPolicy.setJobDB(gJobDB) return S_OK()
def getShifterProxy(shifterType, fileName=False): """ This method returns a shifter's proxy - shifterType : ProductionManager / DataManager... """ if fileName: try: os.makedirs(os.path.dirname(fileName)) except OSError: pass opsHelper = Operations() userName = opsHelper.getValue(cfgPath("Shifter", shifterType, "User"), "") if not userName: return S_ERROR("No shifter User defined for %s" % shifterType) result = CS.getDNForUsername(userName) if not result["OK"]: return result userDN = result["Value"][0] result = CS.findDefaultGroupForDN(userDN) if not result["OK"]: return result defaultGroup = result["Value"] userGroup = opsHelper.getValue(cfgPath("Shifter", shifterType, "Group"), defaultGroup) vomsAttr = CS.getVOMSAttributeForGroup(userGroup) if vomsAttr: gLogger.info("Getting VOMS [%s] proxy for shifter %s@%s (%s)" % (vomsAttr, userName, userGroup, userDN)) result = gProxyManager.downloadVOMSProxyToFile( userDN, userGroup, filePath=fileName, requiredTimeLeft=1200, cacheTime=4 * 43200 ) else: gLogger.info("Getting proxy for shifter %s@%s (%s)" % (userName, userGroup, userDN)) result = gProxyManager.downloadProxyToFile( userDN, userGroup, filePath=fileName, requiredTimeLeft=1200, cacheTime=4 * 43200 ) if not result["OK"]: return result chain = result["chain"] fileName = result["Value"] return S_OK({"DN": userDN, "username": userName, "group": userGroup, "chain": chain, "proxyFile": fileName})
def getTarBallLocation(app, config, dummy_area): """ Get the tar ball location. """ ops = Operations() appName = app[0] appVersion = app[1] appName = appName.lower() app_tar = ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall' % (config, appName, appVersion), '') overwrite = ops.getValue('/AvailableTarBalls/%s/%s/%s/Overwrite' % (config, appName, appVersion), False) md5sum = ops.getValue('/AvailableTarBalls/%s/%s/%s/Md5Sum' % (config, appName, appVersion), '') gLogger.info("Looking for application %s%s for config %s:" % (appName, appVersion, config) ) if not app_tar: gLogger.error('Could not find tar ball for %s %s'%(appName, appVersion)) return S_ERROR('Could not find tar ball for %s %s'%(appName, appVersion)) tarballURL = ops.getValue('/AvailableTarBalls/%s/%s/TarBallURL' % (config, appName), '') if not tarballURL: gLogger.error('Could not find tarballURL in CS for %s %s' % (appName, appVersion)) return S_ERROR('Could not find tarballURL in CS') return S_OK([app_tar, tarballURL, overwrite, md5sum])
def getSteeringFileDirName(systemConfig, application, applicationVersion): """ Locate the path of the steering file directory assigned to the specified application """ ops = Operations() version = ops.getValue('/AvailableTarBalls/%s/%s/%s/Dependencies/steeringfiles/version' % (systemConfig, application, applicationVersion), '') if not version: return S_ERROR("Could not find attached SteeringFile version") TarBall = ops.getValue('/AvailableTarBalls/%s/steeringfiles/%s/TarBall' % (systemConfig, version), '') if not TarBall: return S_ERROR("Could not find tar ball for SteeringFile") mydir = TarBall.replace(".tgz", "").replace(".tar.gz", "") res = getSoftwareFolder(mydir) if not res['OK']: return res mySoftDir = res['Value'] res = check('steeringfiles.%s'%version,'.',[mySoftDir])##check that all the files are there: software is not corrupted. if not res['OK']: return res return S_OK(mySoftDir)
def getSiteSEMapping(gridName=''): """ Returns a dictionary of all sites and their localSEs as a list, e.g. {'LCG.CERN.ch':['CERN-RAW','CERN-RDST',...]} If gridName is specified, result is restricted to that Grid type. """ siteSEMapping = {} gridTypes = gConfig.getSections('Resources/Sites/') if not gridTypes['OK']: gLogger.warn('Problem retrieving sections in /Resources/Sites') return gridTypes gridTypes = gridTypes['Value'] if gridName: if not gridName in gridTypes: return S_ERROR('Could not get sections for /Resources/Sites/%s' % gridName) gridTypes = [gridName] gLogger.debug('Grid Types are: %s' % (', '.join(gridTypes))) for grid in gridTypes: sites = gConfig.getSections('/Resources/Sites/%s' % grid) if not sites['OK']: gLogger.warn('Problem retrieving /Resources/Sites/%s section' % grid) return sites for candidate in sites['Value']: candidateSEs = gConfig.getValue( '/Resources/Sites/%s/%s/SE' % (grid, candidate), []) if candidateSEs: siteSEMapping[candidate] = candidateSEs else: gLogger.debug('No SEs defined for site %s' % candidate) # Add Sites from the SiteLocalSEMapping in the CS cfgLocalSEPath = cfgPath('SiteLocalSEMapping') opsHelper = Operations() result = opsHelper.getOptionsDict(cfgLocalSEPath) if result['OK']: mapping = result['Value'] for site in mapping: ses = opsHelper.getValue(cfgPath(cfgLocalSEPath, site), []) if not ses: continue if gridName: if gridName != site.split('.')[0]: continue if site not in siteSEMapping: siteSEMapping[site] = [] for se in ses: if se not in siteSEMapping[site]: siteSEMapping[site].append(se) return S_OK(siteSEMapping)
def sync(self): """ Main synchronizer method. """ ops = Operations() self.pilotFileServer = ops.getValue("Pilot/pilotFileServer", self.pilotFileServer) if not self.pilotFileServer: gLogger.warn("The /Operations/<Setup>/Pilot/pilotFileServer option is not defined") gLogger.warn("Pilot 3 files won't be updated, and you won't be able to use Pilot 3") gLogger.warn("The Synchronization steps are anyway displayed") gLogger.notice('-- Synchronizing the content of the JSON file %s with the content of the CS --' % self.jsonFile) self.pilotRepo = ops.getValue("Pilot/pilotRepo", self.pilotRepo) self.pilotVORepo = ops.getValue("Pilot/pilotVORepo", self.pilotVORepo) self.projectDir = ops.getValue("Pilot/projectDir", self.projectDir) self.pilotScriptPath = ops.getValue("Pilot/pilotScriptsPath", self.pilotScriptPath) self.pilotVOScriptPath = ops.getValue("Pilot/pilotVOScriptsPath", self.pilotVOScriptPath) result = self._syncJSONFile() if not result['OK']: gLogger.error("Error uploading the pilot file: %s" % result['Message']) return result gLogger.notice('-- Synchronizing the pilot scripts %s with the content of the repository --' % self.pilotRepo) self._syncScripts() return S_OK()
def getSiteSEMapping( gridName = '' ): """ Returns a dictionary of all sites and their localSEs as a list, e.g. {'LCG.CERN.ch':['CERN-RAW','CERN-RDST',...]} If gridName is specified, result is restricted to that Grid type. """ siteSEMapping = {} gridTypes = gConfig.getSections( 'Resources/Sites/' ) if not gridTypes['OK']: gLogger.warn( 'Problem retrieving sections in /Resources/Sites' ) return gridTypes gridTypes = gridTypes['Value'] if gridName: if not gridName in gridTypes: return S_ERROR( 'Could not get sections for /Resources/Sites/%s' % gridName ) gridTypes = [gridName] gLogger.debug( 'Grid Types are: %s' % ( ', '.join( gridTypes ) ) ) for grid in gridTypes: sites = gConfig.getSections( '/Resources/Sites/%s' % grid ) if not sites['OK']: gLogger.warn( 'Problem retrieving /Resources/Sites/%s section' % grid ) return sites for candidate in sites['Value']: candidateSEs = gConfig.getValue( '/Resources/Sites/%s/%s/SE' % ( grid, candidate ), [] ) if candidateSEs: siteSEMapping[candidate] = candidateSEs else: gLogger.debug( 'No SEs defined for site %s' % candidate ) # Add Sites from the SiteLocalSEMapping in the CS cfgLocalSEPath = cfgPath( 'SiteLocalSEMapping' ) opsHelper = Operations() result = opsHelper.getOptionsDict( cfgLocalSEPath ) if result['OK']: mapping = result['Value'] for site in mapping: ses = opsHelper.getValue( cfgPath( cfgLocalSEPath, site ), [] ) if not ses: continue if gridName: if gridName != site.split( '.' )[0]: continue if site not in siteSEMapping: siteSEMapping[site] = [] for se in ses: if se not in siteSEMapping[site]: siteSEMapping[site].append( se ) return S_OK( siteSEMapping )
def getCountryMapping( country ): """ Determines the associated country from the country code""" mappedCountries = [country] opsHelper = Operations() while True: mappedCountry = opsHelper.getValue( '/Countries/%s/AssignedTo' % country, country ) if mappedCountry == country: break elif mappedCountry in mappedCountries: return S_ERROR( 'Circular mapping detected for %s' % country ) else: country = mappedCountry mappedCountries.append( mappedCountry ) return S_OK( mappedCountry )
def getUserRootDir(self): '''get user's initial root directory''' username = self.getUserName() initial = username[:1] vo = getVO() if not vo: vo = 'bes' ops = Operations(vo = vo) user_prefix = ops.getValue('LFNUserPrefix', 'user') basePath = '/' + vo + '/' + user_prefix + '/' + initial + '/' + username return basePath
def resolveDepsTar(sysconfig, appli, appversion): """ Return the dependency tar ball name, if available Uses same parameters as L{resolveDeps}. @return: array of strings """ ops = Operations() deparray = resolveDeps(sysconfig, appli, appversion) depsarray = [] for dep in deparray: dep_tar = ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall' % (sysconfig, dep["app"], dep["version"]), '') if dep_tar: depsarray.append(dep_tar) else: gLogger.error("Dependency %s version %s is not defined in CS, please check !" % (dep["app"], dep["version"])) return depsarray
def __init__(self, dataDir): op = Operations("glast.org") self.stagingArea = dataDir self.listFileStaged = [] self.nbofSEtried = 0 self.userprefix = None self.prefixDest = None self.stagingDest = None SEtemporaryStaging = op.getValue("Pipeline/StorageElementTemporaryStaging", "") self.listSEs = SEtemporaryStaging.split(",") self.SE = None self.log = gLogger.getSubLogger("GridAccess") if self.__getPrefix(): self.log.error("Failed to initialize staging.") raise Exception("Failed to initialize!") self.__pickRandomSE()
def getSEsForCountry( country ): """ Determines the associated SEs from the country code """ mappedCountries = [country] opsHelper = Operations() while True: mappedCountry = opsHelper.getValue( '/Countries/%s/AssignedTo' % country, country ) if mappedCountry == country: break elif mappedCountry in mappedCountries: return S_ERROR( 'Circular mapping detected for %s' % country ) else: country = mappedCountry mappedCountries.append( mappedCountry ) res = opsHelper.getOptionsDict( '/Countries/%s/AssociatedSEs' % country ) if not res['OK']: return S_ERROR( 'Failed to obtain AssociatedSEs for %s' % country ) return S_OK( res['Value'].values() )
def _getProdLogs(): """get production log files from LogSE""" clip = _Params() clip.registerSwitch() Script.parseCommandLine() if not ( clip.logF or clip.logD or clip.prodid ): Script.showHelp() dexit(1) from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations ops = Operations() storageElementName = ops.getValue('/LogStorage/LogSE', 'LogSE') from DIRAC.Resources.Storage.StorageElement import StorageElementItem as StorageElement logSE = StorageElement(storageElementName) if clip.prodid and not ( clip.logD or clip.logF ): result = _getLogFolderFromID( clip ) if not result['OK']: gLogger.error( result['Message'] ) dexit(1) if clip.logD: if not clip.noPromptBeforeDL: res = promptUser('Are you sure you want to get ALL the files in this directory?') if not res['OK']: dexit() choice = res['Value'] if choice.lower()=='n': dexit(0) if isinstance(clip.logD, str): res = logSE.getDirectory(clip.logD, localPath=clip.outputdir) _printErrorReport(res) elif isinstance(clip.logD, list): for logdir in clip.logD: gLogger.notice('Getting log files from '+str(logdir)) res = logSE.getDirectory(logdir, localPath=clip.outputdir) _printErrorReport(res) if clip.logF: res = logSE.getFile(clip.logF, localPath = clip.outputdir) _printErrorReport(res)
class Matcher( object ): """ Logic for matching """ def __init__( self, pilotAgentsDB = None, jobDB = None, tqDB = None, jlDB = None, opsHelper = None ): """ c'tor """ if pilotAgentsDB: self.pilotAgentsDB = pilotAgentsDB else: self.pilotAgentsDB = PilotAgentsDB() if jobDB: self.jobDB = jobDB else: self.jobDB = JobDB() if tqDB: self.tqDB = tqDB else: self.tqDB = TaskQueueDB() if jlDB: self.jlDB = jlDB else: self.jlDB = JobLoggingDB() if opsHelper: self.opsHelper = opsHelper else: self.opsHelper = Operations() self.log = gLogger.getSubLogger( "Matcher" ) self.limiter = Limiter( jobDB = self.jobDB, opsHelper = self.opsHelper ) def selectJob( self, resourceDescription, credDict ): """ Main job selection function to find the highest priority job matching the resource capacity """ startTime = time.time() resourceDict = self._getResourceDict( resourceDescription, credDict ) negativeCond = self.limiter.getNegativeCondForSite( resourceDict['Site'] ) result = self.tqDB.matchAndGetJob( resourceDict, negativeCond = negativeCond ) if not result['OK']: return result result = result['Value'] if not result['matchFound']: self.log.info( "No match found" ) raise RuntimeError( "No match found" ) jobID = result['jobId'] resAtt = self.jobDB.getJobAttributes( jobID, ['OwnerDN', 'OwnerGroup', 'Status'] ) if not resAtt['OK']: raise RuntimeError( 'Could not retrieve job attributes' ) if not resAtt['Value']: raise RuntimeError( "No attributes returned for job" ) if not resAtt['Value']['Status'] == 'Waiting': self.log.error( 'Job matched by the TQ is not in Waiting state', str( jobID ) ) result = self.tqDB.deleteJob( jobID ) if not result[ 'OK' ]: return result raise RuntimeError( "Job %s is not in Waiting state" % str( jobID ) ) self._reportStatus( resourceDict, jobID ) result = self.jobDB.getJobJDL( jobID ) if not result['OK']: raise RuntimeError( "Failed to get the job JDL" ) resultDict = {} resultDict['JDL'] = result['Value'] resultDict['JobID'] = jobID matchTime = time.time() - startTime self.log.info( "Match time: [%s]" % str( matchTime ) ) gMonitor.addMark( "matchTime", matchTime ) # Get some extra stuff into the response returned resOpt = self.jobDB.getJobOptParameters( jobID ) if resOpt['OK']: for key, value in resOpt['Value'].items(): resultDict[key] = value resAtt = self.jobDB.getJobAttributes( jobID, ['OwnerDN', 'OwnerGroup'] ) if not resAtt['OK']: raise RuntimeError( 'Could not retrieve job attributes' ) if not resAtt['Value']: raise RuntimeError( 'No attributes returned for job' ) if self.opsHelper.getValue( "JobScheduling/CheckMatchingDelay", True ): self.limiter.updateDelayCounters( resourceDict['Site'], jobID ) pilotInfoReportedFlag = resourceDict.get( 'PilotInfoReportedFlag', False ) if not pilotInfoReportedFlag: self._updatePilotInfo( resourceDict ) self._updatePilotJobMapping( resourceDict, jobID ) resultDict['DN'] = resAtt['Value']['OwnerDN'] resultDict['Group'] = resAtt['Value']['OwnerGroup'] resultDict['PilotInfoReportedFlag'] = True return resultDict def _getResourceDict( self, resourceDescription, credDict ): """ from resourceDescription to resourceDict (just various mods) """ resourceDict = self._processResourceDescription( resourceDescription ) resourceDict = self._checkCredentials( resourceDict, credDict ) self._checkPilotVersion( resourceDict ) if not self._checkMask( resourceDict ): # Banned destinations can only take Test jobs resourceDict['JobType'] = 'Test' self.log.verbose( "Resource description:" ) for key in resourceDict: self.log.verbose( "%s : %s" % ( key.rjust( 20 ), resourceDict[ key ] ) ) return resourceDict def _processResourceDescription( self, resourceDescription ): """ Check and form the resource description dictionary resourceDescription is a ceDict coming from a JobAgent, for example. """ resourceDict = {} if isinstance( resourceDescription, basestring ): classAdAgent = ClassAd( resourceDescription ) if not classAdAgent.isOK(): raise ValueError( 'Illegal Resource JDL' ) self.log.verbose( classAdAgent.asJDL() ) for name in singleValueDefFields: if classAdAgent.lookupAttribute( name ): if name == 'CPUTime': resourceDict[name] = classAdAgent.getAttributeInt( name ) else: resourceDict[name] = classAdAgent.getAttributeString( name ) for name in multiValueMatchFields: if classAdAgent.lookupAttribute( name ): if name == 'SubmitPool': resourceDict[name] = classAdAgent.getListFromExpression( name ) else: resourceDict[name] = classAdAgent.getAttributeString( name ) # Check if a JobID is requested if classAdAgent.lookupAttribute( 'JobID' ): resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' ) for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization' ): if classAdAgent.lookupAttribute( k ): resourceDict[ k ] = classAdAgent.getAttributeString( k ) else: for name in singleValueDefFields: if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] for name in multiValueMatchFields: if resourceDescription.has_key( name ): resourceDict[name] = resourceDescription[name] if resourceDescription.has_key( 'JobID' ): resourceDict['JobID'] = resourceDescription['JobID'] for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization', 'PilotReference', 'PilotBenchmark', 'PilotInfoReportedFlag' ): if k in resourceDescription: resourceDict[ k ] = resourceDescription[ k ] return resourceDict def _reportStatus( self, resourceDict, jobID ): """ Reports the status of the matched job in jobDB and jobLoggingDB Do not fail if errors happen here """ attNames = ['Status', 'MinorStatus', 'ApplicationStatus', 'Site'] attValues = ['Matched', 'Assigned', 'Unknown', resourceDict['Site']] result = self.jobDB.setJobAttributes( jobID, attNames, attValues ) if not result['OK']: self.log.error( "Problem reporting job status", "setJobAttributes, jobID = %s: %s" % ( jobID, result['Message'] ) ) else: self.log.verbose( "Set job attributes for jobID %s" % jobID ) result = self.jlDB.addLoggingRecord( jobID, status = 'Matched', minor = 'Assigned', source = 'Matcher' ) if not result['OK']: self.log.error( "Problem reporting job status", "addLoggingRecord, jobID = %s: %s" % ( jobID, result['Message'] ) ) else: self.log.verbose( "Added logging record for jobID %s" % jobID ) def _checkMask( self, resourceDict ): """ Check the mask: are we allowed to run normal jobs? FIXME: should we move to site OR SE? """ if not 'Site' in resourceDict: self.log.error( "Missing Site Name in Resource JDL" ) raise RuntimeError( "Missing Site Name in Resource JDL" ) # Get common site mask and check the agent site result = self.jobDB.getSiteMask( siteState = 'Active' ) if not result['OK']: self.log.error( "Internal error", "getSiteMask: %s" % result['Message'] ) raise RuntimeError( "Internal error" ) maskList = result['Value'] if resourceDict['Site'] not in maskList: return False return True def _updatePilotInfo( self, resourceDict ): """ Update pilot information - do not fail if we don't manage to do it """ pilotReference = resourceDict.get( 'PilotReference', '' ) if pilotReference: gridCE = resourceDict.get( 'GridCE', 'Unknown' ) site = resourceDict.get( 'Site', 'Unknown' ) benchmark = resourceDict.get( 'PilotBenchmark', 0.0 ) self.log.verbose( 'Reporting pilot info for %s: gridCE=%s, site=%s, benchmark=%f' % ( pilotReference, gridCE, site, benchmark ) ) result = self.pilotAgentsDB.setPilotStatus( pilotReference, status = 'Running', gridSite = site, destination = gridCE, benchmark = benchmark ) if not result['OK']: self.log.error( "Problem updating pilot information", "; setPilotStatus. pilotReference: %s; %s" % ( pilotReference, result['Message'] ) ) def _updatePilotJobMapping( self, resourceDict, jobID ): """ Update pilot to job mapping information """ pilotReference = resourceDict.get( 'PilotReference', '' ) if pilotReference: result = self.pilotAgentsDB.setCurrentJobID( pilotReference, jobID ) if not result['OK']: self.log.error( "Problem updating pilot information", ";setCurrentJobID. pilotReference: %s; %s" % ( pilotReference, result['Message'] ) ) result = self.pilotAgentsDB.setJobForPilot( jobID, pilotReference, updateStatus = False ) if not result['OK']: self.log.error( "Problem updating pilot information", "; setJobForPilot. pilotReference: %s; %s" % ( pilotReference, result['Message'] ) ) def _checkCredentials( self, resourceDict, credDict ): """ Check if we can get a job given the passed credentials """ if Properties.GENERIC_PILOT in credDict[ 'properties' ]: # You can only match groups in the same VO if credDict[ 'group' ] == "hosts": # for the host case the VirtualOrganization parameter # is mandatory in resourceDict vo = resourceDict.get( 'VirtualOrganization', '' ) else: vo = Registry.getVOForGroup( credDict[ 'group' ] ) result = Registry.getGroupsForVO( vo ) if result[ 'OK' ]: resourceDict[ 'OwnerGroup' ] = result[ 'Value' ] else: raise RuntimeError( result['Message'] ) else: # If it's a private pilot, the DN has to be the same if Properties.PILOT in credDict[ 'properties' ]: self.log.notice( "Setting the resource DN to the credentials DN" ) resourceDict[ 'OwnerDN' ] = credDict[ 'DN' ] # If it's a job sharing. The group has to be the same and just check that the DN (if any) # belongs to the same group elif Properties.JOB_SHARING in credDict[ 'properties' ]: resourceDict[ 'OwnerGroup' ] = credDict[ 'group' ] self.log.notice( "Setting the resource group to the credentials group" ) if 'OwnerDN' in resourceDict and resourceDict[ 'OwnerDN' ] != credDict[ 'DN' ]: ownerDN = resourceDict[ 'OwnerDN' ] result = Registry.getGroupsForDN( resourceDict[ 'OwnerDN' ] ) if not result[ 'OK' ]: raise RuntimeError( result['Message'] ) if credDict[ 'group' ] not in result[ 'Value' ]: # DN is not in the same group! bad boy. self.log.notice( "You cannot request jobs from DN %s. It does not belong to your group!" % ownerDN ) resourceDict[ 'OwnerDN' ] = credDict[ 'DN' ] # Nothing special, group and DN have to be the same else: resourceDict[ 'OwnerDN' ] = credDict[ 'DN' ] resourceDict[ 'OwnerGroup' ] = credDict[ 'group' ] return resourceDict def _checkPilotVersion( self, resourceDict ): """ Check the pilot DIRAC version """ if self.opsHelper.getValue( "Pilot/CheckVersion", True ): if 'ReleaseVersion' not in resourceDict: if not 'DIRACVersion' in resourceDict: raise RuntimeError( 'Version check requested and not provided by Pilot' ) else: pilotVersion = resourceDict['DIRACVersion'] else: pilotVersion = resourceDict['ReleaseVersion'] validVersions = self.opsHelper.getValue( "Pilot/Version", [] ) if validVersions and pilotVersion not in validVersions: raise RuntimeError( 'Pilot version does not match the production version %s not in ( %s )' % \ ( pilotVersion, ",".join( validVersions ) ) ) # Check project if requested validProject = self.opsHelper.getValue( "Pilot/Project", "" ) if validProject: if 'ReleaseProject' not in resourceDict: raise RuntimeError( "Version check requested but expected project %s not received" % validProject ) if resourceDict[ 'ReleaseProject' ] != validProject: raise RuntimeError( "Version check requested but expected project %s != received %s" % ( validProject, resourceDict[ 'ReleaseProject' ] ) )
class JobManifest(object): def __init__(self, manifest=""): self.__manifest = CFG() self.__dirty = False self.__ops = False if manifest: result = self.load(manifest) if not result["OK"]: raise Exception(result["Message"]) def isDirty(self): return self.__dirty def setDirty(self): self.__dirty = True def clearDirty(self): self.__dirty = False def load(self, dataString): """ Auto discover format type based on [ .. ] of JDL """ dataString = dataString.strip() if dataString[0] == "[" and dataString[-1] == "]": return self.loadJDL(dataString) else: return self.loadCFG(dataString) def loadJDL(self, jdlString): """ Load job manifest from JDL format """ result = loadJDLAsCFG(jdlString.strip()) if not result["OK"]: self.__manifest = CFG() return result self.__manifest = result["Value"][0] return S_OK() def loadCFG(self, cfgString): """ Load job manifest from CFG format """ try: self.__manifest.loadFromBuffer(cfgString) except Exception as e: return S_ERROR("Can't load manifest from cfg: %s" % str(e)) return S_OK() def dumpAsCFG(self): return str(self.__manifest) def getAsCFG(self): return self.__manifest.clone() def dumpAsJDL(self): return dumpCFGAsJDL(self.__manifest) def __getCSValue(self, varName, defaultVal=None): if not self.__ops: self.__ops = Operations(group=self.__manifest["OwnerGroup"], setup=self.__manifest["DIRACSetup"]) if varName[0] != "/": varName = "JobDescription/%s" % varName return self.__ops.getValue(varName, defaultVal) def __checkNumericalVar(self, varName, defaultVal, minVal, maxVal): """ Check a numerical var """ initialVal = False if varName not in self.__manifest: varValue = self.__getCSValue("Default%s" % varName, defaultVal) else: varValue = self.__manifest[varName] initialVal = varValue try: varValue = long(varValue) except: return S_ERROR("%s must be a number" % varName) minVal = self.__getCSValue("Min%s" % varName, minVal) maxVal = self.__getCSValue("Max%s" % varName, maxVal) varValue = max(minVal, min(varValue, maxVal)) if initialVal != varValue: self.__manifest.setOption(varName, varValue) return S_OK(varValue) def __checkChoiceVar(self, varName, defaultVal, choices): """ Check a choice var """ initialVal = False if varName not in self.__manifest: varValue = self.__getCSValue("Default%s" % varName, defaultVal) else: varValue = self.__manifest[varName] initialVal = varValue if varValue not in self.__getCSValue("Choices%s" % varName, choices): return S_ERROR("%s is not a valid value for %s" % (varValue, varName)) if initialVal != varValue: self.__manifest.setOption(varName, varValue) return S_OK(varValue) def __checkMultiChoice(self, varName, choices): """ Check a multi choice var """ initialVal = False if varName not in self.__manifest: return S_OK() else: varValue = self.__manifest[varName] initialVal = varValue choices = self.__getCSValue("Choices%s" % varName, choices) for v in List.fromChar(varValue): if v not in choices: return S_ERROR("%s is not a valid value for %s" % (v, varName)) if initialVal != varValue: self.__manifest.setOption(varName, varValue) return S_OK(varValue) def __checkMaxInputData(self, maxNumber): """ Check Maximum Number of Input Data files allowed """ varName = "InputData" if varName not in self.__manifest: return S_OK() varValue = self.__manifest[varName] if len(List.fromChar(varValue)) > maxNumber: return S_ERROR( "Number of Input Data Files (%s) greater than current limit: %s" % (len(List.fromChar(varValue)), maxNumber) ) return S_OK() def __contains__(self, key): """ Check if the manifest has the required key """ return key in self.__manifest def setOptionsFromDict(self, varDict): for k in sorted(varDict): self.setOption(k, varDict[k]) def check(self): """ Check that the manifest is OK """ for k in ["OwnerName", "OwnerDN", "OwnerGroup", "DIRACSetup"]: if k not in self.__manifest: return S_ERROR("Missing var %s in manifest" % k) # Check CPUTime result = self.__checkNumericalVar("CPUTime", 86400, 100, 500000) if not result["OK"]: return result result = self.__checkNumericalVar("Priority", 1, 0, 10) if not result["OK"]: return result allowedSubmitPools = getSubmitPools(self.__manifest["OwnerGroup"]) result = self.__checkMultiChoice("SubmitPools", list(set(allowedSubmitPools))) if not result["OK"]: return result result = self.__checkMultiChoice("PilotTypes", ["private"]) if not result["OK"]: return result maxInputData = Operations().getValue("JobDescription/MaxInputData", 500) result = self.__checkMaxInputData(maxInputData) if not result["OK"]: return result transformationTypes = Operations().getValue("Transformations/DataProcessing", []) result = self.__checkMultiChoice("JobType", ["User", "Test", "Hospital"] + transformationTypes) if not result["OK"]: return result return S_OK() def createSection(self, secName, contents=False): if secName not in self.__manifest: if contents and not isinstance(contents, CFG): return S_ERROR("Contents for section %s is not a cfg object" % secName) self.__dirty = True return S_OK(self.__manifest.createNewSection(secName, contents=contents)) return S_ERROR("Section %s already exists" % secName) def getSection(self, secName): self.__dirty = True sec = self.__manifest[secName] if not sec: return S_ERROR("%s does not exist") return S_OK(sec) def setSectionContents(self, secName, contents): if contents and not isinstance(contents, CFG): return S_ERROR("Contents for section %s is not a cfg object" % secName) self.__dirty = True if secName in self.__manifest: self.__manifest[secName].reset() self.__manifest[secName].mergeWith(contents) else: self.__manifest.createNewSection(secName, contents=contents) def setOption(self, varName, varValue): """ Set a var in job manifest """ self.__dirty = True levels = List.fromChar(varName, "/") cfg = self.__manifest for l in levels[:-1]: if l not in cfg: cfg.createNewSection(l) cfg = cfg[l] cfg.setOption(levels[-1], varValue) def remove(self, opName): levels = List.fromChar(opName, "/") cfg = self.__manifest for l in levels[:-1]: if l not in cfg: return S_ERROR("%s does not exist" % opName) cfg = cfg[l] if cfg.deleteKey(levels[-1]): self.__dirty = True return S_OK() return S_ERROR("%s does not exist" % opName) def getOption(self, varName, defaultValue=None): """ Get a variable from the job manifest """ cfg = self.__manifest return cfg.getOption(varName, defaultValue) def getOptionList(self, section=""): """ Get a list of variables in a section of the job manifest """ cfg = self.__manifest.getRecursive(section) if not cfg or "value" not in cfg: return [] cfg = cfg["value"] return cfg.listOptions() def isOption(self, opName): """ Check if it is a valid option """ return self.__manifest.isOption(opName) def getSectionList(self, section=""): """ Get a list of sections in the job manifest """ cfg = self.__manifest.getRecursive(section) if not cfg or "value" not in cfg: return [] cfg = cfg["value"] return cfg.listSections()