def _getCurrentConfig(self):
    """Return the current system configuration."""
    from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
    gConfig.forceRefresh()

    fullCfg = CFG()
    setup = gConfig.getValue('/DIRAC/Setup', '')
    setupList = gConfig.getSections('/DIRAC/Setups', [])
    if not setupList['OK']:
      return S_ERROR('Could not get /DIRAC/Setups sections')
    setupList = setupList['Value']
    if setup not in setupList:
      return S_ERROR('Setup %s is not in allowed list: %s' % (setup, ', '.join(setupList)))
    serviceSetups = gConfig.getOptionsDict('/DIRAC/Setups/%s' % setup)
    if not serviceSetups['OK']:
      return S_ERROR('Could not get /DIRAC/Setups/%s options' % setup)
    serviceSetups = serviceSetups['Value']  # dict
    for system, setup in serviceSetups.items():
      if self.systems and system not in self.systems:
        continue
      systemCfg = gConfigurationData.remoteCFG.getAsCFG("/Systems/%s/%s" % (system, setup))
      for section in systemCfg.listSections():
        if section not in ('Agents', 'Services', 'Executors'):
          systemCfg.deleteKey(section)

      fullCfg.createNewSection("/%s" % system, contents=systemCfg)

    return S_OK(fullCfg)
Esempio n. 2
0
 def loadFile( self, fileName ):
   try:
     fileCFG = CFG()
     fileCFG.loadFromFile( fileName )
   except IOError:
     self.localCFG = self.localCFG.mergeWith( fileCFG )
     return S_ERROR( "Can't load a cfg file '%s'" % fileName )
   return self.mergeWithLocal( fileCFG )
Esempio n. 3
0
 def mergeWithServer( self ):
   retVal = self.rpcClient.getCompressedData()
   if retVal[ 'OK' ]:
     remoteCFG = CFG()
     remoteCFG.loadFromBuffer( zlib.decompress( retVal[ 'Value' ] ) )
     serverVersion = gConfigurationData.getVersion( remoteCFG )
     self.cfgData = remoteCFG.mergeWith( self.cfgData )
     gConfigurationData.setVersion( serverVersion, self.cfgData )
   return retVal
  def _parseConfigTemplate(self, templatePath, cfg=None):
    """Parse the ConfigTemplate.cfg files.

    :param str templatePath: path to the folder containing a ConfigTemplate.cfg file
    :param CFG cfg: cfg to merge with the systems config
    :returns: CFG object
    """
    cfg = CFG() if cfg is None else cfg

    system = os.path.split(templatePath.rstrip("/"))[1]
    if system.lower().endswith('system'):
      system = system[:-len('System')]

    if self.systems and system not in self.systems:
      return S_OK(cfg)

    templatePath = os.path.join(templatePath, 'ConfigTemplate.cfg')
    if not os.path.exists(templatePath):
      return S_ERROR("File not found: %s" % templatePath)

    loadCfg = CFG()
    loadCfg.loadFromFile(templatePath)

    newCfg = CFG()
    newCfg.createNewSection("/%s" % system, contents=loadCfg)

    cfg = cfg.mergeWith(newCfg)

    return S_OK(cfg)
Esempio n. 5
0
class JobManifest( object ):

  def __init__( self, manifest = "" ):
    self.__manifest = CFG()
    self.__dirty = False
    if manifest:
      result = self.loadManifest( manifest )
      if not result[ 'OK' ]:
        raise Exception( result[ 'Message' ] )

  def isDirty( self ):
    return self.__dirty

  def setDirty( self ):
    self.__dirty = True

  def clearDirty( self ):
    self.__dirty = False

  def load( self, dataString ):
    """
    Auto discover format type based on [ .. ] of JDL
    """
    dataString = dataString.strip()
    if dataString[0] == "[" and dataString[-1] == "]":
      return self.loadJDL( dataString )
    else:
      return self.loadCFG( dataString )

  def loadJDL( self, jdlString ):
    """
    Load job manifest from JDL format
    """
    result = loadJDLAsCFG( jdlString.strip() )
    if not result[ 'OK' ]:
      self.__manifest = CFG()
      return result
    self.__manifest = result[ 'Value' ][0]
    return S_OK()

  def loadCFG( self, cfgString ):
    """
    Load job manifest from CFG format
    """
    try:
      self.__manifest.loadFromBuffer( cfgString )
    except Exception, e:
      return S_ERROR( "Can't load manifest from cfg: %s" % str( e ) )
    return S_OK()
Esempio n. 6
0
 def showTextConfiguration(self):
     response.headers["Content-type"] = "text/plain"
     if "download" in request.params and request.params["download"] in ("yes", "true", "1"):
         version = ""
         try:
             cfg = CFG()
             cfg.loadFromBuffer(session["cfgData"])
             cfg = cfg["DIRAC"]["Configuration"]
             version = ".%s.%s" % (cfg["Name"], cfg["Version"].replace(":", "").replace("-", ""))
         except Exception, e:
             print e
         print 'attachment; filename="cs%s.cfg"' % version.replace(" ", "_")
         response.headers["Content-Disposition"] = 'attachment; filename="cs%s.cfg"' % version.replace(" ", "")
         response.headers["Content-Length"] = len(session["cfgData"])
         response.headers["Content-Transfer-Encoding"] = "Binary"
Esempio n. 7
0
 def appendToRepository( self, repoLocation ):
   if not os.path.exists( repoLocation ):
     gLogger.error( "Secondary repository does not exist", repoLocation )
     return S_ERROR( "Secondary repository does not exist" )
   self.repo = CFG().loadFromFile( repoLocation ).mergeWith( self.repo )
   self._writeRepository( self.location )
   return S_OK()
Esempio n. 8
0
 def __init__( self, rpcClient = False, commiterId = "unknown" ):
   self.commiterTag = "@@-"
   self.commiterId = commiterId
   self.cfgData = CFG()
   self.rpcClient = None
   if rpcClient:
     self.setRPCClient( rpcClient )
Esempio n. 9
0
 def __init__( self, manifest = "" ):
   self.__manifest = CFG()
   self.__dirty = False
   if manifest:
     result = self.loadManifest( manifest )
     if not result[ 'OK' ]:
       raise Exception( result[ 'Message' ] )
Esempio n. 10
0
 def __init__(self, manifest=""):
     self.__manifest = CFG()
     self.__dirty = False
     self.__ops = False
     if manifest:
         result = self.load(manifest)
         if not result["OK"]:
             raise Exception(result["Message"])
Esempio n. 11
0
def __gConfigDefaults(defaultPath):
  """
  Build a cfg from a Default Section
  """
  from DIRAC import gConfig
  cfgDefaults = CFG()
  result = gConfig.getSections(defaultPath)
  if not result['OK']:
    return cfgDefaults
  for name in result['Value']:
    typePath = cfgPath(defaultPath, name)
    cfgDefaults.createNewSection(name)
    result = gConfig.getOptionsDict(typePath)
    if result['OK']:
      optionsDict = result['Value']
      for option, value in optionsDict.items():
        cfgDefaults[name].setOption(option, value)

  return cfgDefaults
Esempio n. 12
0
def loadWebAppCFGFiles():
  """
  Load WebApp/web.cfg definitions
  """
  exts = []
  for ext in CSGlobals.getCSExtensions():
    if ext == "DIRAC":
      continue
    if ext[-5:] != "DIRAC":
      ext = "%sDIRAC" % ext
    if ext != "WebAppDIRAC":
      exts.append( ext )
  exts.append( "DIRAC" )
  exts.append( "WebAppDIRAC" )
  webCFG = CFG()
  for modName in reversed( exts ):
    try:
      modPath = imp.find_module( modName )[1]
    except ImportError:
      continue
    gLogger.verbose( "Found module %s at %s" % ( modName, modPath ) )
    cfgPath = os.path.join( modPath, "WebApp", "web.cfg" )
    if not os.path.isfile( cfgPath ):
      gLogger.verbose( "Inexistant %s" % cfgPath )
      continue
    try:
      modCFG = CFG().loadFromFile( cfgPath )
    except Exception, excp:
      gLogger.error( "Could not load %s: %s" % ( cfgPath, excp ) )
      continue
    gLogger.verbose( "Loaded %s" % cfgPath )
    expl = [ BASECS ]
    while len( expl ):
      current = expl.pop( 0 )
      if not modCFG.isSection( current ):
        continue
      if modCFG.getOption( "%s/AbsoluteDefinition" % current, False ):
        gLogger.verbose( "%s:%s is an absolute definition" % ( modName, current ) )
        try:
          webCFG.deleteKey( current )
        except:
          pass
        modCFG.deleteKey( "%s/AbsoluteDefinition" % current )
      else:
        for sec in modCFG[ current ].listSections():
          expl.append( "%s/%s" % ( current, sec ) )
    #Add the modCFG
    webCFG = webCFG.mergeWith( modCFG )
Esempio n. 13
0
  def setUp( self ):
    self.authMgr = AuthManager( '/Systems/Service/Authorization' )
    cfg = CFG()
    cfg.loadFromBuffer( testSystemsCFG )
    gConfig.loadCFG( cfg )
    cfg.loadFromBuffer( testRegistryCFG )
    gConfig.loadCFG( cfg )

    self.noAuthCredDict = { 'group': 'group_test' }

    self.userCredDict = { 'DN': '/User/test/DN/CN=userA',
                          'group': 'group_test' }

    self.badUserCredDict = { 'DN': '/User/test/DN/CN=userB',
                             'group': 'group_bad' }
    self.hostCredDict = { 'DN': '/User/test/DN/CN=test.hostA.ch',
                          'group': 'hosts' }
    self.badHostCredDict = { 'DN': '/User/test/DN/CN=test.hostB.ch',
                             'group': 'hosts' }
Esempio n. 14
0
 def loadJDL(self, jdlString):
     """
 Load job manifest from JDL format
 """
     result = loadJDLAsCFG(jdlString.strip())
     if not result["OK"]:
         self.__manifest = CFG()
         return result
     self.__manifest = result["Value"][0]
     return S_OK()
Esempio n. 15
0
 def __init__(self, location):
   self.cfg = CFG()
   self.location = location
   self.OK = True
   if os.path.exists(self.location):
     self.cfg.loadFromFile(self.location)
     if not self.cfg.existsKey('Processes'):
       self.cfg.createNewSection('Processes')
   else:
     self.OK = False  
Esempio n. 16
0
 def loadDescriptionFromJDL( self, jdlString ):
   """
   Load job description from JDL format
   """
   result = loadJDLAsCFG( jdlString.strip() )
   if not result[ 'OK' ]:
     self.__description = CFG()
     return result
   self.__description = result[ 'Value' ][0]
   return S_OK()
Esempio n. 17
0
 def __init__( self, loadDefaultCFG = True ):
   lr = LockRing()
   self.threadingEvent = lr.getEvent()
   self.threadingEvent.set()
   self.threadingLock = lr.getLock()
   self.runningThreadsNumber = 0
   self.compressedConfigurationData = ""
   self.configurationPath = "/DIRAC/Configuration"
   self.backupsDir = os.path.join( DIRAC.rootPath, "etc", "csbackup" )
   self._isService = False
   self.localCFG = CFG()
   self.remoteCFG = CFG()
   self.mergedCFG = CFG()
   self.remoteServerList = []
   if loadDefaultCFG:
     defaultCFGFile = os.path.join( DIRAC.rootPath, "etc", "dirac.cfg" )
     gLogger.debug( "dirac.cfg should be at", "%s" % defaultCFGFile )
     retVal = self.loadFile( defaultCFGFile )
     if not retVal[ 'OK' ]:
       gLogger.warn( "Can't load %s file" % defaultCFGFile )
   self.sync()
Esempio n. 18
0
class ProcessList(object):
  """ The ProcessList uses internally the CFG utility to store the processes and their properties.
  """
  def __init__(self, location):
    self.cfg = CFG()
    self.location = location
    self.goodProcessList = True
    if os.path.exists(self.location):
      self.cfg.loadFromFile(self.location)
      if not self.cfg.existsKey('Processes'):
        self.cfg.createNewSection('Processes')
    else:
      self.goodProcessList = False  
      
  def _writeProcessList(self, path):
    """ Write to text
    """
    handle, tmpName = tempfile.mkstemp()
    written = self.cfg.writeToFile(tmpName)
    os.close(handle)
    if not written:
      if os.path.exists(tmpName):
        os.remove(tmpName)
      return written
    if os.path.exists(path):
      LOG.debug("Replacing %s" % path)
    try:
      shutil.move(tmpName, path)
      return True
    except OSError, err:
      LOG.error("Failed to overwrite process list.", err)
      LOG.info("If your process list is corrupted a backup can be found %s" % tmpName)
      return False
Esempio n. 19
0
class JobDescription:

  def __init__( self ):
    self.__description = CFG()
    self.__dirty = False

  def isDirty( self ):
    return self.__dirty

  def loadDescription( self, dataString ):
    """
    Auto discover format type based on [ .. ] of JDL
    """
    dataString = dataString.strip()
    if dataString[0] == "[" and dataString[-1] == "]":
      return self.loadDescriptionFromJDL( dataString )
    else:
      return self.loadDescriptionFromCFG( dataString )

  def loadDescriptionFromJDL( self, jdlString ):
    """
    Load job description from JDL format
    """
    result = loadJDLAsCFG( jdlString.strip() )
    if not result[ 'OK' ]:
      self.__description = CFG()
      return result
    self.__description = result[ 'Value' ][0]
    return S_OK()

  def loadDescriptionFromCFG( self, cfgString ):
    """
    Load job description from CFG format
    """
    try:
      self.__description.loadFromBuffer( cfgString )
    except Exception, e:
      return S_ERROR( "Can't load description from cfg: %s" % str( e ) )
    return S_OK()
  def __saveVomsMapping(self, params):
    sectionPath = "/Registry/VOMS/Mapping"
    sectionCfg = self.getSectionCfg(sectionPath)

    for opt in sectionCfg.listAll():
      if not sectionCfg.isSection(opt):
        self.__configData[ 'cfgData' ].removeOption(sectionPath + "/" + opt)

    configText = ""

    for newOpt in params:
      if newOpt != "op":
        if configText == "":
          configText = newOpt + "=" + params[newOpt] + "\n"
        else:
          configText = configText + newOpt + "=" + params[newOpt] + "\n"

    newCFG = CFG()
    newCFG.loadFromBuffer(configText)
    self.__configData[ 'cfgData' ].mergeSectionFromCFG(sectionPath, newCFG)

    return {"op":"saveVomsMapping", "success": 1}
Esempio n. 21
0
  def execute( self ):
    """ execute
    
    """

    stopAgents = self.findStopAgents()[ 'Value' ]
    if stopAgents:
      self.log.info( 'Aborting, there are stop_agents to be picked' )
      return S_OK()

    pilotVersion = self.opHelper.getValue( 'Pilot/Version', '' )
    if not pilotVersion:
      self.log.error( 'There is no pilot version on the CS' )
      return S_OK()

    pilotVersion = self.getNewestPilotVersion()
    if not pilotVersion[ 'OK' ]:
      self.log.error( pilotVersion[ 'Message' ] )
      return S_ERROR( pilotVersion[ 'Message' ] )
    pilotVersion = pilotVersion[ 'Value' ]
      
    localCFG = CFG()
    
    #load local CFG
    localCFG.loadFromFile( self.cfgToUpdate )
    releaseVersion = localCFG.getRecursive( 'LocalSite/ReleaseVersion' )[ 'value' ]
    
    self.log.info( 'PilotVersion : %s' % pilotVersion )
    self.log.info( 'ReleaseVersion : %s' % releaseVersion )
            
    if LooseVersion( pilotVersion ) > LooseVersion( releaseVersion ):
    
      self.log.info( 'UPDATING %s > %s' % ( pilotVersion, releaseVersion ) )
    
      localCFG.setOption( 'LocalSite/ReleaseVersion', pilotVersion )
      localCFG.writeToFile( self.cfgToUpdate )  
      
      self.touchStopAgents()
    
    else:
      
      self.log.info( 'Nothing to do' )
    
    return S_OK()     
Esempio n. 22
0
 def __init__( self, repository = None ):
   self.location = repository
   if not self.location:
     if "HOME" in os.environ:
       self.location = '%s/.dirac.repo.rep' % os.environ['HOME']
     else:
       self.location = '%s/.dirac.repo.rep' % os.getcwd()
   self.repo = CFG()
   if os.path.exists( self.location ):
     self.repo.loadFromFile( self.location )
     if not self.repo.existsKey( 'Jobs' ):
       self.repo.createNewSection( 'Jobs' )
   else:
     self.repo.createNewSection( 'Jobs' )
   self.OK = True
   written = self._writeRepository( self.location )
   if not written:
     self.OK = False
Esempio n. 23
0
  def _loadWebAppCFGFiles(self, extension):
    """
    Load WebApp/web.cfg definitions

    :param str extension: the module name of the extension of WebAppDirac for example: LHCbWebDIRAC
    """
    exts = [extension, "WebAppDIRAC"]
    webCFG = CFG()
    for modName in reversed(exts):
      cfgPath = os.path.join(self.__params.destination, "%s/WebApp" % modName, "web.cfg")
      if not os.path.isfile(cfgPath):
        gLogger.verbose("Web configuration file %s does not exists!" % cfgPath)
        continue
      try:
        modCFG = CFG().loadFromFile(cfgPath)
      except Exception, excp:
        gLogger.error("Could not load %s: %s" % (cfgPath, excp))
        continue
      gLogger.verbose("Loaded %s" % cfgPath)
      expl = ["/WebApp"]
      while len(expl):
        current = expl.pop(0)
        if not modCFG.isSection(current):
          continue
        if modCFG.getOption("%s/AbsoluteDefinition" % current, False):
          gLogger.verbose("%s:%s is an absolute definition" % (modName, current))
          try:
            webCFG.deleteKey(current)
          except:
            pass
          modCFG.deleteKey("%s/AbsoluteDefinition" % current)
        else:
          for sec in modCFG[current].listSections():
            expl.append("%s/%s" % (current, sec))
      # Add the modCFG
      webCFG = webCFG.mergeWith(modCFG)
Esempio n. 24
0
def getComputingElementDefaults(ceName="", ceType="", cfg=None, currentSectionPath=""):
    """
  Return cfgDefaults with defaults for the given CEs defined either in arguments or in the provided cfg
  """
    cesCfg = CFG()
    if cfg:
        try:
            cesCfg.loadFromFile(cfg)
            cesPath = cfgInstallPath("ComputingElements")
            if cesCfg.isSection(cesPath):
                for section in cfgPathToList(cesPath):
                    cesCfg = cesCfg[section]
        except:
            return CFG()

    # Overwrite the cfg with Command line arguments
    if ceName:
        if not cesCfg.isSection(ceName):
            cesCfg.createNewSection(ceName)
        if currentSectionPath:
            # Add Options from Command Line
            optionsDict = __getExtraOptions(currentSectionPath)
            for name, value in optionsDict.items():
                cesCfg[ceName].setOption(name, value)
        if ceType:
            cesCfg[ceName].setOption("CEType", ceType)

    ceDefaultSection = cfgPath(defaultSection("ComputingElements"))
    # Load Default for the given type from Central configuration is defined
    ceDefaults = __gConfigDefaults(ceDefaultSection)
    for ceName in cesCfg.listSections():
        if "CEType" in cesCfg[ceName]:
            ceType = cesCfg[ceName]["CEType"]
            if ceType in ceDefaults:
                for option in ceDefaults[ceType].listOptions():
                    if option not in cesCfg[ceName]:
                        cesCfg[ceName].setOption(option, ceDefaults[ceType][option])

    return cesCfg
Esempio n. 25
0
def getComputingElementDefaults(ceName='', ceType='', cfg=None, currentSectionPath=''):
  """
  Return cfgDefaults with defaults for the given CEs defined either in arguments or in the provided cfg
  """
  cesCfg = CFG()
  if cfg:
    try:
      cesCfg.loadFromFile(cfg)
      cesPath = cfgInstallPath('ComputingElements')
      if cesCfg.isSection(cesPath):
        for section in cfgPathToList(cesPath):
          cesCfg = cesCfg[section]
    except BaseException:
      return CFG()

  # Overwrite the cfg with Command line arguments
  if ceName:
    if not cesCfg.isSection(ceName):
      cesCfg.createNewSection(ceName)
    if currentSectionPath:
      # Add Options from Command Line
      optionsDict = __getExtraOptions(currentSectionPath)
      for name, value in optionsDict.items():
        cesCfg[ceName].setOption(name, value)  # pylint: disable=no-member
    if ceType:
      cesCfg[ceName].setOption('CEType', ceType)  # pylint: disable=no-member

  ceDefaultSection = cfgPath(defaultSection('ComputingElements'))
  # Load Default for the given type from Central configuration is defined
  ceDefaults = __gConfigDefaults(ceDefaultSection)
  for ceName in cesCfg.listSections():
    if 'CEType' in cesCfg[ceName]:
      ceType = cesCfg[ceName]['CEType']
      if ceType in ceDefaults:
        for option in ceDefaults[ceType].listOptions():  # pylint: disable=no-member
          if option not in cesCfg[ceName]:
            cesCfg[ceName].setOption(option, ceDefaults[ceType][option])  # pylint: disable=unsubscriptable-object

  return cesCfg
Esempio n. 26
0
class JobRepository(object):
    def __init__(self, repository=None):
        self.location = repository
        if not self.location:
            if "HOME" in os.environ:
                self.location = '%s/.dirac.repo.rep' % os.environ['HOME']
            else:
                self.location = '%s/.dirac.repo.rep' % os.getcwd()
        self.repo = CFG()
        if os.path.exists(self.location):
            self.repo.loadFromFile(self.location)
            if not self.repo.existsKey('Jobs'):
                self.repo.createNewSection('Jobs')
        else:
            self.repo.createNewSection('Jobs')
        self.OK = True
        written = self._writeRepository(self.location)
        if not written:
            self.OK = False

    def isOK(self):
        return self.OK

    def readRepository(self):
        return S_OK(self.repo.getAsDict('Jobs'))

    def writeRepository(self, alternativePath=None):
        destination = self.location
        if alternativePath:
            destination = alternativePath
        written = self._writeRepository(destination)
        if not written:
            return S_ERROR("Failed to write repository")
        return S_OK(destination)

    def resetRepository(self, jobIDs=[]):
        if not jobIDs:
            jobs = self.readRepository()['Value']
            jobIDs = jobs.keys()
        paramDict = {'State': 'Submitted', 'Retrieved': 0, 'OutputData': 0}
        for jobID in jobIDs:
            self._writeJob(jobID, paramDict, True)
        self._writeRepository(self.location)
        return S_OK()

    def _writeRepository(self, path):
        handle, tmpName = tempfile.mkstemp()
        written = self.repo.writeToFile(tmpName)
        os.close(handle)
        if not written:
            if os.path.exists(tmpName):
                os.remove(tmpName)
            return written
        if os.path.exists(path):
            gLogger.debug("Replacing %s" % path)
        try:
            shutil.move(tmpName, path)
            return True
        except Exception as x:
            gLogger.error("Failed to overwrite repository.", x)
            gLogger.info(
                "If your repository is corrupted a backup can be found %s" %
                tmpName)
            return False

    def appendToRepository(self, repoLocation):
        if not os.path.exists(repoLocation):
            gLogger.error("Secondary repository does not exist", repoLocation)
            return S_ERROR("Secondary repository does not exist")
        self.repo = CFG().loadFromFile(repoLocation).mergeWith(self.repo)
        self._writeRepository(self.location)
        return S_OK()

    def addJob(self,
               jobID,
               state='Submitted',
               retrieved=0,
               outputData=0,
               update=False):
        paramDict = {
            'State': state,
            'Time': self._getTime(),
            'Retrieved': int(retrieved),
            'OutputData': outputData
        }
        self._writeJob(jobID, paramDict, update)
        self._writeRepository(self.location)
        return S_OK(jobID)

    def updateJob(self, jobID, paramDict):
        if self._existsJob(jobID):
            paramDict['Time'] = self._getTime()
            self._writeJob(jobID, paramDict, True)
            self._writeRepository(self.location)
        return S_OK()

    def updateJobs(self, jobDict):
        for jobID, paramDict in jobDict.items():
            if self._existsJob(jobID):
                paramDict['Time'] = self._getTime()
                self._writeJob(jobID, paramDict, True)
        self._writeRepository(self.location)
        return S_OK()

    def _getTime(self):
        runtime = time.ctime()
        return runtime.replace(" ", "_")

    def _writeJob(self, jobID, paramDict, update):
        jobID = str(jobID)
        jobExists = self._existsJob(jobID)
        if jobExists and (not update):
            gLogger.warn("Job exists and not overwriting")
            return S_ERROR("Job exists and not overwriting")
        if not jobExists:
            self.repo.createNewSection('Jobs/%s' % jobID)
        for key, value in paramDict.items():
            self.repo.setOption('Jobs/%s/%s' % (jobID, key), value)
        return S_OK()

    def removeJob(self, jobID):
        res = self.repo['Jobs'].deleteKey(str(jobID))  #pylint: disable=no-member
        if res:
            self._writeRepository(self.location)
        return S_OK()

    def existsJob(self, jobID):
        return S_OK(self._existsJob(jobID))

    def _existsJob(self, jobID):
        return self.repo.isSection('Jobs/%s' % jobID)

    def getLocation(self):
        return S_OK(self.location)

    def getSize(self):
        return S_OK(len(self.repo.getAsDict('Jobs')))
  def __editItem(self, params):

    ret = self.__deleteItem(params)
    if ret["success"] == 1:
      ret = self.__addItem(params)
      ret["op"] = "editItem"
      return ret
    ret["op"] = "editItem"
    return ret

    sectionPath = "/Registry/"
    configText = ""
    if params["type"] == "users":

      sectionPath = sectionPath + "Users"
      if params["dn"].strip() != "":
        configText = "DN = " + params["dn"].strip() + "\n"

      if params["ca"].strip() != "":
        configText = configText + "CA = " + params["ca"].strip() + "\n"

      if params["email"].strip() != "":
        configText = configText + "Email = " + params["email"].strip()

    elif params["type"] == "groups":

      sectionPath = sectionPath + "Groups"
      if params["users"].strip() != "":
        configText = "Users = " + params["users"].strip() + "\n"

      if params["properties"].strip() != "":
        configText = configText + "Properties = " + params["properties"].strip() + "\n"

      if str(params["jobshare"]).strip() != "":
        configText = configText + "JobShare = " + str(params["jobshare"]) + "\n"

      if params["autouploadproxy"].strip() != "":
        configText = configText + "AutoUploadProxy = " + params["autouploadproxy"].strip() + "\n"

      if params["autouploadpilotproxy"].strip() != "":
        configText = configText + "AutoUploadPilotProxy = " + params["autouploadpilotproxy"].strip() + "\n"

      if params["autoaddvoms"].strip() != "":
        configText = configText + "AutoAddVOMS = " + params["autoaddvoms"].strip()

    elif params["type"] == "hosts":

      sectionPath = sectionPath + "Hosts"
      if params["dn"].strip() != "":
        configText = "DN = " + params["dn"].strip() + "\n"

      if params["properties"].strip() != "":
        configText = configText + "Properties = " + params["properties"].strip()

    sectionPath = sectionPath + "/" + params["name"]

#   deleting the options underneath
    sectionCfg = self.getSectionCfg(sectionPath)

    for opt in sectionCfg.listAll():
      print "deleting " + opt + "\n"
      self.__configData[ 'cfgData' ].removeOption(sectionPath + "/" + opt)

    newCFG = CFG()
    newCFG.loadFromBuffer(configText)
    self.__configData[ 'cfgData' ].mergeSectionFromCFG(sectionPath, newCFG)
    return {"success":1, "op": "editItem"}
Esempio n. 28
0
class JobManifest(object):
    def __init__(self, manifest=""):
        self.__manifest = CFG()
        self.__dirty = False
        self.__ops = False
        if manifest:
            result = self.load(manifest)
            if not result["OK"]:
                raise Exception(result["Message"])

    def isDirty(self):
        return self.__dirty

    def setDirty(self):
        self.__dirty = True

    def clearDirty(self):
        self.__dirty = False

    def load(self, dataString):
        """
    Auto discover format type based on [ .. ] of JDL
    """
        dataString = dataString.strip()
        if dataString[0] == "[" and dataString[-1] == "]":
            return self.loadJDL(dataString)
        else:
            return self.loadCFG(dataString)

    def loadJDL(self, jdlString):
        """
    Load job manifest from JDL format
    """
        result = loadJDLAsCFG(jdlString.strip())
        if not result["OK"]:
            self.__manifest = CFG()
            return result
        self.__manifest = result["Value"][0]
        return S_OK()

    def loadCFG(self, cfgString):
        """
    Load job manifest from CFG format
    """
        try:
            self.__manifest.loadFromBuffer(cfgString)
        except Exception as e:
            return S_ERROR("Can't load manifest from cfg: %s" % str(e))
        return S_OK()

    def dumpAsCFG(self):
        return str(self.__manifest)

    def getAsCFG(self):
        return self.__manifest.clone()

    def dumpAsJDL(self):
        return dumpCFGAsJDL(self.__manifest)

    def __getCSValue(self, varName, defaultVal=None):
        if not self.__ops:
            self.__ops = Operations(group=self.__manifest["OwnerGroup"], setup=self.__manifest["DIRACSetup"])
        if varName[0] != "/":
            varName = "JobDescription/%s" % varName
        return self.__ops.getValue(varName, defaultVal)

    def __checkNumericalVar(self, varName, defaultVal, minVal, maxVal):
        """
    Check a numerical var
    """
        initialVal = False
        if varName not in self.__manifest:
            varValue = self.__getCSValue("Default%s" % varName, defaultVal)
        else:
            varValue = self.__manifest[varName]
            initialVal = varValue
        try:
            varValue = long(varValue)
        except:
            return S_ERROR("%s must be a number" % varName)
        minVal = self.__getCSValue("Min%s" % varName, minVal)
        maxVal = self.__getCSValue("Max%s" % varName, maxVal)
        varValue = max(minVal, min(varValue, maxVal))
        if initialVal != varValue:
            self.__manifest.setOption(varName, varValue)
        return S_OK(varValue)

    def __checkChoiceVar(self, varName, defaultVal, choices):
        """
    Check a choice var
    """
        initialVal = False
        if varName not in self.__manifest:
            varValue = self.__getCSValue("Default%s" % varName, defaultVal)
        else:
            varValue = self.__manifest[varName]
            initialVal = varValue
        if varValue not in self.__getCSValue("Choices%s" % varName, choices):
            return S_ERROR("%s is not a valid value for %s" % (varValue, varName))
        if initialVal != varValue:
            self.__manifest.setOption(varName, varValue)
        return S_OK(varValue)

    def __checkMultiChoice(self, varName, choices):
        """
    Check a multi choice var
    """
        initialVal = False
        if varName not in self.__manifest:
            return S_OK()
        else:
            varValue = self.__manifest[varName]
            initialVal = varValue
        choices = self.__getCSValue("Choices%s" % varName, choices)
        for v in List.fromChar(varValue):
            if v not in choices:
                return S_ERROR("%s is not a valid value for %s" % (v, varName))
        if initialVal != varValue:
            self.__manifest.setOption(varName, varValue)
        return S_OK(varValue)

    def __checkMaxInputData(self, maxNumber):
        """
    Check Maximum Number of Input Data files allowed
    """
        varName = "InputData"
        if varName not in self.__manifest:
            return S_OK()
        varValue = self.__manifest[varName]
        if len(List.fromChar(varValue)) > maxNumber:
            return S_ERROR(
                "Number of Input Data Files (%s) greater than current limit: %s"
                % (len(List.fromChar(varValue)), maxNumber)
            )
        return S_OK()

    def __contains__(self, key):
        """ Check if the manifest has the required key
    """
        return key in self.__manifest

    def setOptionsFromDict(self, varDict):
        for k in sorted(varDict):
            self.setOption(k, varDict[k])

    def check(self):
        """
    Check that the manifest is OK
    """
        for k in ["OwnerName", "OwnerDN", "OwnerGroup", "DIRACSetup"]:
            if k not in self.__manifest:
                return S_ERROR("Missing var %s in manifest" % k)

        # Check CPUTime
        result = self.__checkNumericalVar("CPUTime", 86400, 100, 500000)
        if not result["OK"]:
            return result

        result = self.__checkNumericalVar("Priority", 1, 0, 10)
        if not result["OK"]:
            return result

        allowedSubmitPools = getSubmitPools(self.__manifest["OwnerGroup"])
        result = self.__checkMultiChoice("SubmitPools", list(set(allowedSubmitPools)))
        if not result["OK"]:
            return result

        result = self.__checkMultiChoice("PilotTypes", ["private"])
        if not result["OK"]:
            return result

        maxInputData = Operations().getValue("JobDescription/MaxInputData", 500)
        result = self.__checkMaxInputData(maxInputData)
        if not result["OK"]:
            return result

        transformationTypes = Operations().getValue("Transformations/DataProcessing", [])
        result = self.__checkMultiChoice("JobType", ["User", "Test", "Hospital"] + transformationTypes)
        if not result["OK"]:
            return result
        return S_OK()

    def createSection(self, secName, contents=False):
        if secName not in self.__manifest:
            if contents and not isinstance(contents, CFG):
                return S_ERROR("Contents for section %s is not a cfg object" % secName)
            self.__dirty = True
            return S_OK(self.__manifest.createNewSection(secName, contents=contents))
        return S_ERROR("Section %s already exists" % secName)

    def getSection(self, secName):
        self.__dirty = True
        sec = self.__manifest[secName]
        if not sec:
            return S_ERROR("%s does not exist")
        return S_OK(sec)

    def setSectionContents(self, secName, contents):
        if contents and not isinstance(contents, CFG):
            return S_ERROR("Contents for section %s is not a cfg object" % secName)
        self.__dirty = True
        if secName in self.__manifest:
            self.__manifest[secName].reset()
            self.__manifest[secName].mergeWith(contents)
        else:
            self.__manifest.createNewSection(secName, contents=contents)

    def setOption(self, varName, varValue):
        """
    Set a var in job manifest
    """
        self.__dirty = True
        levels = List.fromChar(varName, "/")
        cfg = self.__manifest
        for l in levels[:-1]:
            if l not in cfg:
                cfg.createNewSection(l)
            cfg = cfg[l]
        cfg.setOption(levels[-1], varValue)

    def remove(self, opName):
        levels = List.fromChar(opName, "/")
        cfg = self.__manifest
        for l in levels[:-1]:
            if l not in cfg:
                return S_ERROR("%s does not exist" % opName)
            cfg = cfg[l]
        if cfg.deleteKey(levels[-1]):
            self.__dirty = True
            return S_OK()
        return S_ERROR("%s does not exist" % opName)

    def getOption(self, varName, defaultValue=None):
        """
     Get a variable from the job manifest
    """
        cfg = self.__manifest
        return cfg.getOption(varName, defaultValue)

    def getOptionList(self, section=""):
        """
    Get a list of variables in a section of the job manifest
    """
        cfg = self.__manifest.getRecursive(section)
        if not cfg or "value" not in cfg:
            return []
        cfg = cfg["value"]
        return cfg.listOptions()

    def isOption(self, opName):
        """
    Check if it is a valid option
    """
        return self.__manifest.isOption(opName)

    def getSectionList(self, section=""):
        """
    Get a list of sections in the job manifest
    """
        cfg = self.__manifest.getRecursive(section)
        if not cfg or "value" not in cfg:
            return []
        cfg = cfg["value"]
        return cfg.listSections()
Esempio n. 29
0
 def toCFG(self):
     """ Get the full description of the file in CFG format
 """
     oCFG = CFG()
     strippedLFN = self.lfn.replace('/', '&&')
     oCFG.createNewSection(strippedLFN)
     oCFG.setOption('%s/Status' % (strippedLFN), self.status)
     oCFG.setOption('%s/Size' % (strippedLFN), self.size)
     oCFG.setOption('%s/GUID' % (strippedLFN), self.guid)
     oCFG.setOption('%s/Checksum' % (strippedLFN), self.checksum)
     #TODO: still have to include the CFG from the replica objects
     if self.catalogReplicas:
         oCFG.createNewSection('%s/CatalogReplicas' % strippedLFN)
         for replica in self.catalogReplicas:
             pass
             #  rCFG.mergeWith(CFG().loadFromBuffer(replica.toCFG()['Value']))
     return S_OK(str(oCFG))
Esempio n. 30
0
    def _loadWebAppCFGFiles(self, extension):
        """
    Load WebApp/web.cfg definitions

    :param str extension: the module name of the extension of WebAppDirac for example: LHCbWebDIRAC
    """
        exts = [extension, "WebAppDIRAC"]
        webCFG = CFG()
        for modName in reversed(exts):
            cfgPath = os.path.join(self.__params.destination,
                                   "%s/WebApp" % modName, "web.cfg")
            if not os.path.isfile(cfgPath):
                gLogger.verbose("Web configuration file %s does not exists!" %
                                cfgPath)
                continue
            try:
                modCFG = CFG().loadFromFile(cfgPath)
            except Exception, excp:
                gLogger.error("Could not load %s: %s" % (cfgPath, excp))
                continue
            gLogger.verbose("Loaded %s" % cfgPath)
            expl = ["/WebApp"]
            while len(expl):
                current = expl.pop(0)
                if not modCFG.isSection(current):
                    continue
                if modCFG.getOption("%s/AbsoluteDefinition" % current, False):
                    gLogger.verbose("%s:%s is an absolute definition" %
                                    (modName, current))
                    try:
                        webCFG.deleteKey(current)
                    except:
                        pass
                    modCFG.deleteKey("%s/AbsoluteDefinition" % current)
                else:
                    for sec in modCFG[current].listSections():
                        expl.append("%s/%s" % (current, sec))
            # Add the modCFG
            webCFG = webCFG.mergeWith(modCFG)
Esempio n. 31
0
class ConfigurationData:

  def __init__( self, loadDefaultCFG = True ):
    lr = LockRing()
    self.threadingEvent = lr.getEvent()
    self.threadingEvent.set()
    self.threadingLock = lr.getLock()
    self.runningThreadsNumber = 0
    self.compressedConfigurationData = ""
    self.configurationPath = "/DIRAC/Configuration"
    self.backupsDir = os.path.join( DIRAC.rootPath, "etc", "csbackup" )
    self._isService = False
    self.localCFG = CFG()
    self.remoteCFG = CFG()
    self.mergedCFG = CFG()
    self.remoteServerList = []
    if loadDefaultCFG:
      defaultCFGFile = os.path.join( DIRAC.rootPath, "etc", "dirac.cfg" )
      gLogger.debug( "dirac.cfg should be at", "%s" % defaultCFGFile )
      retVal = self.loadFile( defaultCFGFile )
      if not retVal[ 'OK' ]:
        gLogger.warn( "Can't load %s file" % defaultCFGFile )
    self.sync()

  def getBackupDir( self ):
    return self.backupsDir

  def sync( self ):
    gLogger.debug( "Updating configuration internals" )
    self.mergedCFG = self.remoteCFG.mergeWith( self.localCFG )
    self.remoteServerList = []
    localServers = self.extractOptionFromCFG( "%s/Servers" % self.configurationPath,
                                        self.localCFG,
                                        disableDangerZones = True )
    if localServers:
      self.remoteServerList.extend( List.fromChar( localServers, "," ) )
    remoteServers = self.extractOptionFromCFG( "%s/Servers" % self.configurationPath,
                                        self.remoteCFG,
                                        disableDangerZones = True )
    if remoteServers:
      self.remoteServerList.extend( List.fromChar( remoteServers, "," ) )
    self.remoteServerList = List.uniqueElements( self.remoteServerList )
    self.compressedConfigurationData = zlib.compress( str( self.remoteCFG ), 9 )

  def loadFile( self, fileName ):
    try:
      fileCFG = CFG()
      fileCFG.loadFromFile( fileName )
    except IOError:
      self.localCFG = self.localCFG.mergeWith( fileCFG )
      return S_ERROR( "Can't load a cfg file '%s'" % fileName )
    return self.mergeWithLocal( fileCFG )

  def mergeWithLocal( self, extraCFG ):
    self.lock()
    try:
      self.localCFG = self.localCFG.mergeWith( extraCFG )
      self.unlock()
      gLogger.debug( "CFG merged" )
    except Exception, e:
      self.unlock()
      return S_ERROR( "Cannot merge with new cfg: %s" % str( e ) )
    self.sync()
    return S_OK()
Esempio n. 32
0
    def setUpClass(cls):
        cls.failed = False

        # Add configuration
        cfg = CFG()
        cfg.loadFromBuffer(diracTestCACFG)
        gConfig.loadCFG(cfg)
        cfg.loadFromBuffer(userCFG)
        gConfig.loadCFG(cfg)

        # Prepare CA
        lines = []
        cfgDict = {}
        cls.caPath = os.path.join(certsPath, 'ca')
        cls.caConfigFile = os.path.join(cls.caPath, 'openssl_config_ca.cnf')
        # Save original configuration file
        shutil.copyfile(cls.caConfigFile, cls.caConfigFile + 'bak')
        # Parse
        fields = [
            'dir', 'database', 'serial', 'new_certs_dir', 'private_key',
            'certificate'
        ]
        with open(cls.caConfigFile, "r") as caCFG:
            for line in caCFG:
                if re.findall('=', re.sub(r'#.*', '', line)):
                    field = re.sub(r'#.*', '',
                                   line).replace(' ',
                                                 '').rstrip().split('=')[0]
                    line = 'dir = %s #PUT THE RIGHT DIR HERE!\n' % (
                        cls.caPath) if field == 'dir' else line
                    val = re.sub(r'#.*', '',
                                 line).replace(' ', '').rstrip().split('=')[1]
                    if field in fields:
                        for i in fields:
                            if cfgDict.get(i):
                                val = val.replace('$%s' % i, cfgDict[i])
                        cfgDict[field] = val
                        if not cfgDict[field]:
                            cls.failed = '%s have empty value in %s' % (
                                field, cls.caConfigFile)
                lines.append(line)
        with open(cls.caConfigFile, "w") as caCFG:
            caCFG.writelines(lines)
        for field in fields:
            if field not in cfgDict.keys():
                cls.failed = '%s value is absent in %s' % (field,
                                                           cls.caConfigFile)
        cls.hostCert = os.path.join(certsPath, 'host/hostcert.pem')
        cls.hostKey = os.path.join(certsPath, 'host/hostkey.pem')
        cls.caCert = cfgDict['certificate']
        cls.caKey = cfgDict['private_key']
        os.chmod(cls.caKey, stat.S_IREAD)
        # Check directory for new certificates
        cls.newCertDir = cfgDict['new_certs_dir']
        if not os.path.exists(cls.newCertDir):
            os.makedirs(cls.newCertDir)
        for f in os.listdir(cls.newCertDir):
            os.remove(os.path.join(cls.newCertDir, f))
        # Empty the certificate database
        cls.index = cfgDict['database']
        with open(cls.index, 'w') as indx:
            indx.write('')
        # Write down serial
        cls.serial = cfgDict['serial']
        with open(cls.serial, 'w') as serialFile:
            serialFile.write('1000')

        # Create temporaly directory for users certificates
        cls.userDir = tempfile.mkdtemp(dir=certsPath)

        # Create user certificates
        for userName in ['no_user', 'user', 'user_1', 'user_2', 'user_3']:
            userConf = """[ req ]
        default_bits           = 4096
        encrypt_key            = yes
        distinguished_name     = req_dn
        prompt                 = no
        req_extensions         = v3_req
        [ req_dn ]
        C                      = CC
        O                      = DN
        0.O                    = DIRAC
        CN                     = %s
        [ v3_req ]
        # Extensions for client certificates (`man x509v3_config`).
        nsComment = "OpenSSL Generated Client Certificate"
        keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment
        extendedKeyUsage = clientAuth
        """ % (userName)
            userConfFile = os.path.join(cls.userDir, userName + '.cnf')
            userReqFile = os.path.join(cls.userDir, userName + '.req')
            userKeyFile = os.path.join(cls.userDir, userName + '.key.pem')
            userCertFile = os.path.join(cls.userDir, userName + '.cert.pem')
            with open(userConfFile, "w") as f:
                f.write(userConf)
            status, output = commands.getstatusoutput(
                'openssl genrsa -out %s' % userKeyFile)
            if status:
                gLogger.error(output)
                exit()
            gLogger.debug(output)
            os.chmod(userKeyFile, stat.S_IREAD)
            status, output = commands.getstatusoutput(
                'openssl req -config %s -key %s -new -out %s' %
                (userConfFile, userKeyFile, userReqFile))
            if status:
                gLogger.error(output)
                exit()
            gLogger.debug(output)
            cmd = 'openssl ca -config %s -extensions usr_cert -batch -days 375 -in %s -out %s'
            cmd = cmd % (cls.caConfigFile, userReqFile, userCertFile)
            status, output = commands.getstatusoutput(cmd)
            if status:
                gLogger.error(output)
                exit()
            gLogger.debug(output)

        # Result
        status, output = commands.getstatusoutput('ls -al %s' % cls.userDir)
        if status:
            gLogger.error(output)
            exit()
        gLogger.debug('User certificates:\n', output)
  def __editItem(self, params):

    ret = self.__deleteItem(params)
    if ret["success"] == 1:
      ret = self.__addItem(params)
      ret["op"] = "editItem"
      return ret
    ret["op"] = "editItem"
    return ret

    sectionPath = "/Registry/"
    configText = ""
    if params["type"] == "users":

      sectionPath = sectionPath + "Users"
      if params["dn"].strip() != "":
        configText = "DN = " + params["dn"].strip() + "\n"

      if params["ca"].strip() != "":
        configText = configText + "CA = " + params["ca"].strip() + "\n"

      if params["email"].strip() != "":
        configText = configText + "Email = " + params["email"].strip()

    elif params["type"] == "groups":

      sectionPath = sectionPath + "Groups"
      if params["users"].strip() != "":
        configText = "Users = " + params["users"].strip() + "\n"

      if params["properties"].strip() != "":
        configText = configText + "Properties = " + params["properties"].strip() + "\n"

      if str(params["jobshare"]).strip() != "":
        configText = configText + "JobShare = " + str(params["jobshare"]) + "\n"

      if params["autouploadproxy"].strip() != "":
        configText = configText + "AutoUploadProxy = " + params["autouploadproxy"].strip() + "\n"

      if params["autouploadpilotproxy"].strip() != "":
        configText = configText + "AutoUploadPilotProxy = " + params["autouploadpilotproxy"].strip() + "\n"

      if params["autoaddvoms"].strip() != "":
        configText = configText + "AutoAddVOMS = " + params["autoaddvoms"].strip()

    elif params["type"] == "hosts":

      sectionPath = sectionPath + "Hosts"
      if params["dn"].strip() != "":
        configText = "DN = " + params["dn"].strip() + "\n"

      if params["properties"].strip() != "":
        configText = configText + "Properties = " + params["properties"].strip()

    sectionPath = sectionPath + "/" + params["name"]

#   deleting the options underneath
    sectionCfg = self.getSectionCfg(sectionPath)

    for opt in sectionCfg.listAll():
      print "deleting " + opt + "\n"
      self.__configData[ 'cfgData' ].removeOption(sectionPath + "/" + opt)

    newCFG = CFG()
    newCFG.loadFromBuffer(configText)
    self.__configData[ 'cfgData' ].mergeSectionFromCFG(sectionPath, newCFG)
    return {"success":1, "op": "editItem"}
  def __addItem(self, params):

    sectionPath = "/Registry/"
    configText = ""
    if params["type"] == "users":

      sectionPath = sectionPath + "Users"
      if params["dn"].strip() != "":
        configText = "DN = " + params["dn"].strip() + "\n"

      if params["ca"].strip() != "":
        configText = configText + "CA = " + params["ca"].strip() + "\n"

      if params["email"].strip() != "":
        configText = configText + "Email = " + params["email"].strip()

    elif params["type"] == "groups":

      sectionPath = sectionPath + "Groups"
      if params["users"].strip() != "":
        configText = "Users = " + params["users"].strip() + "\n"

      if params["properties"].strip() != "":
        configText = configText + "Properties = " + params["properties"].strip() + "\n"

      if str(params["jobshare"]).strip() != "":
        configText = configText + "JobShare = " + str(params["jobshare"]) + "\n"

      if params["autouploadproxy"].strip() != "":
        configText = configText + "AutoUploadProxy = " + params["autouploadproxy"].strip() + "\n"

      if params["autouploadpilotproxy"].strip() != "":
        configText = configText + "AutoUploadPilotProxy = " + params["autouploadpilotproxy"].strip() + "\n"

      if params["autoaddvoms"].strip() != "":
        configText = configText + "AutoAddVOMS = " + params["autoaddvoms"].strip()

    elif params["type"] == "hosts":

      sectionPath = sectionPath + "Hosts"
      if params["dn"].strip() != "":
        configText = "DN = " + params["dn"].strip() + "\n"

      if params["properties"].strip() != "":
        configText = configText + "Properties = " + params["properties"].strip()

    elif params["type"] == "voms":

      sectionPath = sectionPath + "VOMS/Servers"

    elif params["type"] == "servers":

      sectionPath = sectionPath + "VOMS/Servers/" + params["vom"]
      if params["dn"].strip() != "":
        configText = "DN = " + params["dn"].strip() + "\n"

      if params["port"].strip() != "":
        configText = configText + "Port = " + params["port"].strip() + "\n"

      if params["ca"].strip() != "":
        configText = configText + "CA = " + params["ca"].strip()

    sectionPath = sectionPath + "/" + params["name"]

    if self.__configData[ 'cfgData' ].createSection(sectionPath):
      cfgData = self.__configData[ 'cfgData' ].getCFG()
      newCFG = CFG()
      newCFG.loadFromBuffer(configText)
      self.__configData[ 'cfgData' ].mergeSectionFromCFG(sectionPath, newCFG)
      return {"success":1, "op": "addItem"}
    else:
      return {"success":0, "op":"addItem", "message":"Section can't be created. It already exists?"}
Esempio n. 35
0
# $HeadURL$
__RCSID__ = "7b8878b (2009-11-05 19:40:01 +0000) Adria Casajus <*****@*****.**>"

from dirac import DIRAC
from DIRAC.Core.Utilities.CFG import CFG

DIRAC.gLogger.initialize('test_gConfig','/testSectionDebug')

testconfig = '%s/DIRAC/ConfigurationSystem/test/test.cfg' % DIRAC.rootPath
dumpconfig = '%s/DIRAC/ConfigurationSystem/test/dump.cfg' % DIRAC.rootPath

cfg1 = CFG()
cfg1.loadFromFile( testconfig )

fd = file( testconfig )
cfg1String = fd.read()
fd.close()

cfg2 = CFG()
cfg2.loadFromBuffer( cfg1.serialize() )

cfg3 = cfg1.mergeWith( cfg2 )

testList = [{ 'method'    : DIRAC.gConfig.loadFile,
              'arguments' : ( testconfig, ),
              'output'    : {'OK': True, 'Value': ''}
            },
            { 'method'    : DIRAC.gConfig.dumpLocalCFGToFile,
              'arguments' : ( dumpconfig, ),
              'output'    : {'OK': True, 'Value': ''}
            },
Esempio n. 36
0
class JobManifest(object):

  def __init__(self, manifest=""):
    self.__manifest = CFG()
    self.__dirty = False
    self.__ops = False
    if manifest:
      result = self.load(manifest)
      if not result['OK']:
        raise Exception(result['Message'])

  def isDirty(self):
    return self.__dirty

  def setDirty(self):
    self.__dirty = True

  def clearDirty(self):
    self.__dirty = False

  def load(self, dataString):
    """
    Auto discover format type based on [ .. ] of JDL
    """
    dataString = dataString.strip()
    if dataString[0] == "[" and dataString[-1] == "]":
      return self.loadJDL(dataString)
    else:
      return self.loadCFG(dataString)

  def loadJDL(self, jdlString):
    """
    Load job manifest from JDL format
    """
    result = loadJDLAsCFG(jdlString.strip())
    if not result['OK']:
      self.__manifest = CFG()
      return result
    self.__manifest = result['Value'][0]
    return S_OK()

  def loadCFG(self, cfgString):
    """
    Load job manifest from CFG format
    """
    try:
      self.__manifest.loadFromBuffer(cfgString)
    except Exception as e:
      return S_ERROR("Can't load manifest from cfg: %s" % str(e))
    return S_OK()

  def dumpAsCFG(self):
    return str(self.__manifest)

  def getAsCFG(self):
    return self.__manifest.clone()

  def dumpAsJDL(self):
    return dumpCFGAsJDL(self.__manifest)

  def __getCSValue(self, varName, defaultVal=None):
    if not self.__ops:
      self.__ops = Operations(group=self.__manifest['OwnerGroup'], setup=self.__manifest['DIRACSetup'])
    if varName[0] != "/":
      varName = "JobDescription/%s" % varName
    return self.__ops.getValue(varName, defaultVal)

  def __checkNumericalVar(self, varName, defaultVal, minVal, maxVal):
    """
    Check a numerical var
    """
    initialVal = False
    if varName not in self.__manifest:
      varValue = self.__getCSValue("Default%s" % varName, defaultVal)
    else:
      varValue = self.__manifest[varName]
      initialVal = varValue
    try:
      varValue = long(varValue)
    except BaseException:
      return S_ERROR("%s must be a number" % varName)
    minVal = self.__getCSValue("Min%s" % varName, minVal)
    maxVal = self.__getCSValue("Max%s" % varName, maxVal)
    varValue = max(minVal, min(varValue, maxVal))
    if initialVal != varValue:
      self.__manifest.setOption(varName, varValue)
    return S_OK(varValue)

  def __checkChoiceVar(self, varName, defaultVal, choices):
    """
    Check a choice var
    """
    initialVal = False
    if varName not in self.__manifest:
      varValue = self.__getCSValue("Default%s" % varName, defaultVal)
    else:
      varValue = self.__manifest[varName]
      initialVal = varValue
    if varValue not in self.__getCSValue("Choices%s" % varName, choices):
      return S_ERROR("%s is not a valid value for %s" % (varValue, varName))
    if initialVal != varValue:
      self.__manifest.setOption(varName, varValue)
    return S_OK(varValue)

  def __checkMultiChoice(self, varName, choices):
    """
    Check a multi choice var
    """
    initialVal = False
    if varName not in self.__manifest:
      return S_OK()
    else:
      varValue = self.__manifest[varName]
      initialVal = varValue
    choices = self.__getCSValue("Choices%s" % varName, choices)
    for v in List.fromChar(varValue):
      if v not in choices:
        return S_ERROR("%s is not a valid value for %s" % (v, varName))
    if initialVal != varValue:
      self.__manifest.setOption(varName, varValue)
    return S_OK(varValue)

  def __checkMaxInputData(self, maxNumber):
    """
    Check Maximum Number of Input Data files allowed
    """
    varName = "InputData"
    if varName not in self.__manifest:
      return S_OK()
    varValue = self.__manifest[varName]
    if len(List.fromChar(varValue)) > maxNumber:
      return S_ERROR('Number of Input Data Files (%s) greater than current limit: %s' %
                     (len(List.fromChar(varValue)), maxNumber))
    return S_OK()

  def __contains__(self, key):
    """ Check if the manifest has the required key
    """
    return key in self.__manifest

  def setOptionsFromDict(self, varDict):
    for k in sorted(varDict):
      self.setOption(k, varDict[k])

  def check(self):
    """
    Check that the manifest is OK
    """
    for k in ['OwnerName', 'OwnerDN', 'OwnerGroup', 'DIRACSetup']:
      if k not in self.__manifest:
        return S_ERROR("Missing var %s in manifest" % k)

    # Check CPUTime
    result = self.__checkNumericalVar("CPUTime", 86400, 100, 500000)
    if not result['OK']:
      return result

    result = self.__checkNumericalVar("Priority", 1, 0, 10)
    if not result['OK']:
      return result

    allowedSubmitPools = getSubmitPools(self.__manifest['OwnerGroup'])
    result = self.__checkMultiChoice("SubmitPools", list(set(allowedSubmitPools)))
    if not result['OK']:
      return result

    result = self.__checkMultiChoice("PilotTypes", ['private'])
    if not result['OK']:
      return result

    maxInputData = Operations().getValue("JobDescription/MaxInputData", 500)
    result = self.__checkMaxInputData(maxInputData)
    if not result['OK']:
      return result

    transformationTypes = Operations().getValue("Transformations/DataProcessing", [])
    result = self.__checkMultiChoice("JobType", ['User', 'Test', 'Hospital'] + transformationTypes)
    if not result['OK']:
      return result
    return S_OK()

  def createSection(self, secName, contents=False):
    if secName not in self.__manifest:
      if contents and not isinstance(contents, CFG):
        return S_ERROR("Contents for section %s is not a cfg object" % secName)
      self.__dirty = True
      return S_OK(self.__manifest.createNewSection(secName, contents=contents))
    return S_ERROR("Section %s already exists" % secName)

  def getSection(self, secName):
    self.__dirty = True
    if secName not in self.__manifest:
      return S_ERROR("%s does not exist" % secName)
    sec = self.__manifest[secName]
    if not sec:
      return S_ERROR("%s section empty" % secName)
    return S_OK(sec)

  def setSectionContents(self, secName, contents):
    if contents and not isinstance(contents, CFG):
      return S_ERROR("Contents for section %s is not a cfg object" % secName)
    self.__dirty = True
    if secName in self.__manifest:
      self.__manifest[secName].reset()
      self.__manifest[secName].mergeWith(contents)
    else:
      self.__manifest.createNewSection(secName, contents=contents)

  def setOption(self, varName, varValue):
    """
    Set a var in job manifest
    """
    self.__dirty = True
    levels = List.fromChar(varName, "/")
    cfg = self.__manifest
    for l in levels[:-1]:
      if l not in cfg:
        cfg.createNewSection(l)
      cfg = cfg[l]
    cfg.setOption(levels[-1], varValue)

  def remove(self, opName):
    levels = List.fromChar(opName, "/")
    cfg = self.__manifest
    for l in levels[:-1]:
      if l not in cfg:
        return S_ERROR("%s does not exist" % opName)
      cfg = cfg[l]
    if cfg.deleteKey(levels[-1]):
      self.__dirty = True
      return S_OK()
    return S_ERROR("%s does not exist" % opName)

  def getOption(self, varName, defaultValue=None):
    """
     Get a variable from the job manifest
    """
    cfg = self.__manifest
    return cfg.getOption(varName, defaultValue)

  def getOptionList(self, section=""):
    """
    Get a list of variables in a section of the job manifest
    """
    cfg = self.__manifest.getRecursive(section)
    if not cfg or 'value' not in cfg:
      return []
    cfg = cfg['value']
    return cfg.listOptions()

  def isOption(self, opName):
    """
    Check if it is a valid option
    """
    return self.__manifest.isOption(opName)

  def getSectionList(self, section=""):
    """
    Get a list of sections in the job manifest
    """
    cfg = self.__manifest.getRecursive(section)
    if not cfg or 'value' not in cfg:
      return []
    cfg = cfg['value']
    return cfg.listSections()
Esempio n. 37
0
    def execute(self):
        """The JobAgent execution method.
    """
        if self.jobCount:
            # Temporary mechanism to pass a shutdown message to the agent
            if os.path.exists('/var/lib/dirac_drain'):
                return self.__finish('Node is being drained by an operator')
            # Only call timeLeft utility after a job has been picked up
            self.log.info('Attempting to check CPU time left for filling mode')
            if self.fillingMode:
                if self.timeLeftError:
                    self.log.warn(self.timeLeftError)
                    return self.__finish(self.timeLeftError)
                self.log.info('%s normalized CPU units remaining in slot' %
                              (self.timeLeft))
                if self.timeLeft <= self.minimumTimeLeft:
                    return self.__finish('No more time left')
                # Need to update the Configuration so that the new value is published in the next matching request
                result = self.computingElement.setCPUTimeLeft(
                    cpuTimeLeft=self.timeLeft)
                if not result['OK']:
                    return self.__finish(result['Message'])

                # Update local configuration to be used by submitted job wrappers
                localCfg = CFG()
                if self.extraOptions:
                    localConfigFile = os.path.join('.', self.extraOptions)
                else:
                    localConfigFile = os.path.join(rootPath, "etc",
                                                   "dirac.cfg")
                localCfg.loadFromFile(localConfigFile)
                if not localCfg.isSection('/LocalSite'):
                    localCfg.createNewSection('/LocalSite')
                localCfg.setOption('/LocalSite/CPUTimeLeft', self.timeLeft)
                localCfg.writeToFile(localConfigFile)

            else:
                return self.__finish('Filling Mode is Disabled')

        self.log.verbose('Job Agent execution loop')
        result = self.computingElement.available()
        if not result['OK']:
            self.log.info('Resource is not available')
            self.log.info(result['Message'])
            return self.__finish('CE Not Available')

        self.log.info(result['Message'])

        ceInfoDict = result['CEInfoDict']
        runningJobs = ceInfoDict.get("RunningJobs")
        availableSlots = result['Value']

        if not availableSlots:
            if runningJobs:
                self.log.info('No available slots with %d running jobs' %
                              runningJobs)
                return S_OK('Job Agent cycle complete with %d running jobs' %
                            runningJobs)
            else:
                self.log.info('CE is not available')
                return self.__finish('CE Not Available')

        result = self.computingElement.getDescription()
        if not result['OK']:
            return result

        # We can have several prioritized job retrieval strategies
        if isinstance(result['Value'], dict):
            ceDictList = [result['Value']]
        elif isinstance(result['Value'], list):
            ceDictList = result['Value']

        for ceDict in ceDictList:

            # Add pilot information
            gridCE = gConfig.getValue('LocalSite/GridCE', 'Unknown')
            if gridCE != 'Unknown':
                ceDict['GridCE'] = gridCE
            if 'PilotReference' not in ceDict:
                ceDict['PilotReference'] = str(self.pilotReference)
            ceDict['PilotBenchmark'] = self.cpuFactor
            ceDict['PilotInfoReportedFlag'] = self.pilotInfoReportedFlag

            # Add possible job requirements
            result = gConfig.getOptionsDict('/AgentJobRequirements')
            if result['OK']:
                requirementsDict = result['Value']
                ceDict.update(requirementsDict)
                self.log.info('Requirements:', requirementsDict)

            self.log.verbose(ceDict)
            start = time.time()
            jobRequest = MatcherClient().requestJob(ceDict)
            matchTime = time.time() - start
            self.log.info('MatcherTime = %.2f (s)' % (matchTime))
            if jobRequest['OK']:
                break

        self.stopAfterFailedMatches = self.am_getOption(
            'StopAfterFailedMatches', self.stopAfterFailedMatches)

        if not jobRequest['OK']:
            if re.search('No match found', jobRequest['Message']):
                self.log.notice('Job request OK: %s' % (jobRequest['Message']))
                self.matchFailedCount += 1
                if self.matchFailedCount > self.stopAfterFailedMatches:
                    return self.__finish(
                        'Nothing to do for more than %d cycles' %
                        self.stopAfterFailedMatches)
                return S_OK(jobRequest['Message'])
            elif jobRequest['Message'].find("seconds timeout") != -1:
                self.log.error('Timeout while requesting job',
                               jobRequest['Message'])
                self.matchFailedCount += 1
                if self.matchFailedCount > self.stopAfterFailedMatches:
                    return self.__finish(
                        'Nothing to do for more than %d cycles' %
                        self.stopAfterFailedMatches)
                return S_OK(jobRequest['Message'])
            elif jobRequest['Message'].find(
                    "Pilot version does not match") != -1:
                errorMsg = 'Pilot version does not match the production version'
                self.log.error(errorMsg,
                               jobRequest['Message'].replace(errorMsg, ''))
                return S_ERROR(jobRequest['Message'])
            else:
                self.log.notice('Failed to get jobs: %s' %
                                (jobRequest['Message']))
                self.matchFailedCount += 1
                if self.matchFailedCount > self.stopAfterFailedMatches:
                    return self.__finish(
                        'Nothing to do for more than %d cycles' %
                        self.stopAfterFailedMatches)
                return S_OK(jobRequest['Message'])

        # Reset the Counter
        self.matchFailedCount = 0

        matcherInfo = jobRequest['Value']
        if not self.pilotInfoReportedFlag:
            # Check the flag after the first access to the Matcher
            self.pilotInfoReportedFlag = matcherInfo.get(
                'PilotInfoReportedFlag', False)
        jobID = matcherInfo['JobID']
        matcherParams = ['JDL', 'DN', 'Group']
        for param in matcherParams:
            if param not in matcherInfo:
                self.__report(jobID, 'Failed',
                              'Matcher did not return %s' % (param))
                return self.__finish('Matcher Failed')
            elif not matcherInfo[param]:
                self.__report(jobID, 'Failed',
                              'Matcher returned null %s' % (param))
                return self.__finish('Matcher Failed')
            else:
                self.log.verbose('Matcher returned %s = %s ' %
                                 (param, matcherInfo[param]))

        jobJDL = matcherInfo['JDL']
        jobGroup = matcherInfo['Group']
        ownerDN = matcherInfo['DN']

        optimizerParams = {}
        for key in matcherInfo:
            if key not in matcherParams:
                optimizerParams[key] = matcherInfo[key]

        parameters = self.__getJDLParameters(jobJDL)
        if not parameters['OK']:
            self.__report(jobID, 'Failed', 'Could Not Extract JDL Parameters')
            self.log.warn(parameters['Message'])
            return self.__finish('JDL Problem')

        params = parameters['Value']
        if 'JobID' not in params:
            msg = 'Job has not JobID defined in JDL parameters'
            self.__report(jobID, 'Failed', msg)
            self.log.warn(msg)
            return self.__finish('JDL Problem')
        else:
            jobID = params['JobID']

        if 'JobType' not in params:
            self.log.warn('Job has no JobType defined in JDL parameters')
            jobType = 'Unknown'
        else:
            jobType = params['JobType']

        if 'CPUTime' not in params:
            self.log.warn(
                'Job has no CPU requirement defined in JDL parameters')

        # Job requirement for a number of processors
        processors = int(params.get('NumberOfProcessors', 1))
        wholeNode = 'WholeNode' in params

        if self.extraOptions:
            params['Arguments'] += ' ' + self.extraOptions
            params['ExtraOptions'] = self.extraOptions

        self.log.verbose('Job request successful: \n', jobRequest['Value'])
        self.log.info('Received JobID=%s, JobType=%s' % (jobID, jobType))
        self.log.info('OwnerDN: %s JobGroup: %s' % (ownerDN, jobGroup))
        self.jobCount += 1
        try:
            jobReport = JobReport(jobID, 'JobAgent@%s' % self.siteName)
            jobReport.setJobParameter('MatcherServiceTime',
                                      str(matchTime),
                                      sendFlag=False)

            if 'BOINC_JOB_ID' in os.environ:
                # Report BOINC environment
                for thisp in ('BoincUserID', 'BoincHostID',
                              'BoincHostPlatform', 'BoincHostName'):
                    jobReport.setJobParameter(thisp,
                                              gConfig.getValue(
                                                  '/LocalSite/%s' % thisp,
                                                  'Unknown'),
                                              sendFlag=False)

            jobReport.setJobStatus('Matched', 'Job Received by Agent')
            result = self.__setupProxy(ownerDN, jobGroup)
            if not result['OK']:
                return self.__rescheduleFailedJob(
                    jobID, result['Message'], self.stopOnApplicationFailure)
            proxyChain = result.get('Value')

            # Save the job jdl for external monitoring
            self.__saveJobJDLRequest(jobID, jobJDL)

            software = self.__checkInstallSoftware(jobID, params, ceDict)
            if not software['OK']:
                self.log.error('Failed to install software for job',
                               '%s' % (jobID))
                errorMsg = software['Message']
                if not errorMsg:
                    errorMsg = 'Failed software installation'
                return self.__rescheduleFailedJob(
                    jobID, errorMsg, self.stopOnApplicationFailure)

            self.log.debug('Before %sCE submitJob()' % (self.ceName))
            result = self.__submitJob(jobID, params, ceDict, optimizerParams,
                                      proxyChain, processors, wholeNode)
            if not result['OK']:
                self.__report(jobID, 'Failed', result['Message'])
                return self.__finish(result['Message'])
            elif 'PayloadFailed' in result:
                # Do not keep running and do not overwrite the Payload error
                message = 'Payload execution failed with error code %s' % result[
                    'PayloadFailed']
                if self.stopOnApplicationFailure:
                    return self.__finish(message,
                                         self.stopOnApplicationFailure)
                else:
                    self.log.info(message)

            self.log.debug('After %sCE submitJob()' % (self.ceName))
        except Exception as subExcept:  # pylint: disable=broad-except
            self.log.exception("Exception in submission",
                               "",
                               lException=subExcept,
                               lExcInfo=True)
            return self.__rescheduleFailedJob(
                jobID, 'Job processing failed with exception',
                self.stopOnApplicationFailure)

        # Sum all times but the last one (elapsed_time) and remove times at init (is this correct?)
        cpuTime = sum(os.times()[:-1]) - sum(self.initTimes[:-1])

        result = self.timeLeftUtil.getTimeLeft(cpuTime, processors)
        if result['OK']:
            self.timeLeft = result['Value']
        else:
            if result['Message'] != 'Current batch system is not supported':
                self.timeLeftError = result['Message']
            else:
                # if the batch system is not defined, use the process time and the CPU normalization defined locally
                self.timeLeft = self.__getCPUTimeLeft()

        return S_OK('Job Agent cycle complete')
Esempio n. 38
0
class JobDescription(object):
    def __init__(self):
        self.__description = CFG()
        self.__dirty = False

    def isDirty(self):
        return self.__dirty

    def loadDescription(self, dataString):
        """
    Auto discover format type based on [ .. ] of JDL
    """
        dataString = dataString.strip()
        if dataString[0] == "[" and dataString[-1] == "]":
            return self.loadDescriptionFromJDL(dataString)
        else:
            return self.loadDescriptionFromCFG(dataString)

    def loadDescriptionFromJDL(self, jdlString):
        """
    Load job description from JDL format
    """
        result = loadJDLAsCFG(jdlString.strip())
        if not result['OK']:
            self.__description = CFG()
            return result
        self.__description = result['Value'][0]
        return S_OK()

    def loadDescriptionFromCFG(self, cfgString):
        """
    Load job description from CFG format
    """
        try:
            self.__description.loadFromBuffer(cfgString)
        except Exception as e:
            return S_ERROR("Can't load description from cfg: %s" % str(e))
        return S_OK()

    def dumpDescriptionAsCFG(self):
        return str(self.__description)

    def dumpDescriptionAsJDL(self):
        return dumpCFGAsJDL(self.__description)

    def __checkNumericalVarInDescription(self, varName, defaultVal, minVal,
                                         maxVal):
        """
    Check a numerical var
    """
        initialVal = 0
        if varName not in self.__description:
            varValue = Operations().getValue(
                "JobDescription/Default%s" % varName, defaultVal)
        else:
            varValue = self.__description[varName]
            initialVal = varValue
        try:
            varValue = long(varValue)
        except BaseException:
            return S_ERROR("%s must be a number" % varName)
        minVal = Operations().getValue("JobDescription/Min%s" % varName,
                                       minVal)
        maxVal = Operations().getValue("JobDescription/Max%s" % varName,
                                       maxVal)
        varValue = max(minVal, min(varValue, maxVal))
        if initialVal != varValue:
            self.__description.setOption(varName, varValue)
        return S_OK(varValue)

    def __checkChoiceVarInDescription(self, varName, defaultVal, choices):
        """
    Check a choice var
    """
        initialVal = False
        if varName not in self.__description:
            varValue = Operations().getValue(
                "JobDescription/Default%s" % varName, defaultVal)
        else:
            varValue = self.__description[varName]
            initialVal = varValue
        if varValue not in Operations().getValue(
                "JobDescription/Choices%s" % varName, choices):
            return S_ERROR("%s is not a valid value for %s" %
                           (varValue, varName))
        if initialVal != varValue:
            self.__description.setOption(varName, varValue)
        return S_OK(varValue)

    def __checkMultiChoiceInDescription(self, varName, choices):
        """
    Check a multi choice var
    """
        initialVal = False
        if varName not in self.__description:
            return S_OK()
        else:
            varValue = self.__description[varName]
            initialVal = varValue
        choices = Operations().getValue("JobDescription/Choices%s" % varName,
                                        choices)
        for v in List.fromChar(varValue):
            if v not in choices:
                return S_ERROR("%s is not a valid value for %s" % (v, varName))
        if initialVal != varValue:
            self.__description.setOption(varName, varValue)
        return S_OK(varValue)

    def __checkMaxInputData(self, maxNumber):
        """
    Check Maximum Number of Input Data files allowed
    """
        varName = "InputData"
        if varName not in self.__description:
            return S_OK()
        varValue = self.__description[varName]
        if len(List.fromChar(varValue)) > maxNumber:
            return S_ERROR(
                'Number of Input Data Files (%s) greater than current limit: %s'
                % (len(List.fromChar(varValue)), maxNumber))
        return S_OK()

    def setVarsFromDict(self, varDict):
        for k in sorted(varDict):
            self.setVar(k, varDict[k])

    def checkDescription(self):
        """
    Check that the description is OK
    """
        for k in ['OwnerName', 'OwnerDN', 'OwnerGroup', 'DIRACSetup']:
            if k not in self.__description:
                return S_ERROR("Missing var %s in description" % k)
        # Check CPUTime
        result = self.__checkNumericalVarInDescription("CPUTime", 86400, 0,
                                                       500000)
        if not result['OK']:
            return result
        result = self.__checkNumericalVarInDescription("Priority", 1, 0, 10)
        if not result['OK']:
            return result

        allowedSubmitPools = getSubmitPools(self.__description['OwnerGroup'])
        result = self.__checkMultiChoiceInDescription(
            "SubmitPools", list(set(allowedSubmitPools)))
        if not result['OK']:
            return result

        result = self.__checkMultiChoiceInDescription(
            "SubmitPools", list(set(allowedSubmitPools)))
        if not result['OK']:
            return result
        result = self.__checkMultiChoiceInDescription("PilotTypes",
                                                      ['private'])
        if not result['OK']:
            return result
        maxInputData = Operations().getValue("JobDescription/MaxInputData",
                                             500)
        result = self.__checkMaxInputData(maxInputData)
        if not result['OK']:
            return result
        transformationTypes = Operations().getValue(
            "Transformations/DataProcessing", [])
        result = self.__checkMultiChoiceInDescription(
            "JobType", ['User', 'Test', 'Hospital'] + transformationTypes)
        return S_OK()

    def setVar(self, varName, varValue):
        """
    Set a var in job description
    """
        self.__dirty = True
        levels = List.fromChar(varName, "/")
        cfg = self.__description
        for l in levels[:-1]:
            if l not in cfg:
                cfg.createNewSection(l)
            cfg = cfg[l]
        cfg.setOption(levels[-1], varValue)

    def getVar(self, varName, defaultValue=None):
        cfg = self.__description
        return cfg.getOption(varName, defaultValue)

    def getOptionList(self, section=""):
        cfg = self.__description.getRecursive(section)
        if not cfg or 'value' not in cfg:
            return []
        cfg = cfg['value']
        return cfg.listOptions()

    def getSectionList(self, section=""):
        cfg = self.__description.getRecursive(section)
        if not cfg or 'value' not in cfg:
            return []
        cfg = cfg['value']
        return cfg.listSections()
Esempio n. 39
0
 def __init__(self):
     self.__description = CFG()
     self.__dirty = False
  def __addItem(self, params):

    sectionPath = "/Registry/"
    configText = ""
    if params["type"] == "users":

      sectionPath = sectionPath + "Users"
      if params["dn"].strip() != "":
        configText = "DN = " + params["dn"].strip() + "\n"

      if params["ca"].strip() != "":
        configText = configText + "CA = " + params["ca"].strip() + "\n"

      if params["email"].strip() != "":
        configText = configText + "Email = " + params["email"].strip()

    elif params["type"] == "groups":

      sectionPath = sectionPath + "Groups"
      if params["users"].strip() != "":
        configText = "Users = " + params["users"].strip() + "\n"

      if params["properties"].strip() != "":
        configText = configText + "Properties = " + params["properties"].strip() + "\n"

      if str(params["jobshare"]).strip() != "":
        configText = configText + "JobShare = " + str(params["jobshare"]) + "\n"

      if params["autouploadproxy"].strip() != "":
        configText = configText + "AutoUploadProxy = " + params["autouploadproxy"].strip() + "\n"

      if params["autouploadpilotproxy"].strip() != "":
        configText = configText + "AutoUploadPilotProxy = " + params["autouploadpilotproxy"].strip() + "\n"

      if params["autoaddvoms"].strip() != "":
        configText = configText + "AutoAddVOMS = " + params["autoaddvoms"].strip()

    elif params["type"] == "hosts":

      sectionPath = sectionPath + "Hosts"
      if params["dn"].strip() != "":
        configText = "DN = " + params["dn"].strip() + "\n"

      if params["properties"].strip() != "":
        configText = configText + "Properties = " + params["properties"].strip()

    elif params["type"] == "voms":

      sectionPath = sectionPath + "VOMS/Servers"

    elif params["type"] == "servers":

      sectionPath = sectionPath + "VOMS/Servers/" + params["vom"]
      if params["dn"].strip() != "":
        configText = "DN = " + params["dn"].strip() + "\n"

      if params["port"].strip() != "":
        configText = configText + "Port = " + params["port"].strip() + "\n"

      if params["ca"].strip() != "":
        configText = configText + "CA = " + params["ca"].strip()

    sectionPath = sectionPath + "/" + params["name"]

    if self.__configData[ 'cfgData' ].createSection(sectionPath):
      cfgData = self.__configData[ 'cfgData' ].getCFG()
      newCFG = CFG()
      newCFG.loadFromBuffer(configText)
      self.__configData[ 'cfgData' ].mergeSectionFromCFG(sectionPath, newCFG)
      return {"success":1, "op": "addItem"}
    else:
      return {"success":0, "op":"addItem", "message":"Section can't be created. It already exists?"}