def __getBackendsFromCFG(self, cfgPath):
"""
Get backends from the configuration and register them in LoggingRoot.
This is the new way to get the backends providing a general configuration.
:params cfgPath: string of the configuration path
"""
# We have to put the import line here to avoid a dependancy loop
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC import gConfig
# get the second last string representing the component type in the configuration
# example : 'Agents', 'Services'
component = cfgPath.split("/")[-2]
operation = Operations()
# Search desired backends in the component
desiredBackends = gConfig.getValue("%s/%s" % (cfgPath, 'LogBackends'),
[])
if not desiredBackends:
# Search desired backends in the operation section according to the component type
desiredBackends = operation.getValue(
"Logging/Default%sBackends" % component, [])
if not desiredBackends:
# Search desired backends in the operation section
desiredBackends = operation.getValue("Logging/DefaultBackends",
[])
if not desiredBackends:
# Default value
desiredBackends = ['stdout']
return desiredBackends
def configure( self, csSection, submitPool ):
"""
Here goes common configuration for all PilotDirectors
"""
self.configureFromSection( csSection )
self.reloadConfiguration( csSection, submitPool )
# Get the defaults for the Setup where the Director is running
opsHelper = Operations()
self.installVersion = opsHelper.getValue( cfgPath( 'Pilot', 'Version' ), [ self.installVersion ] )[0]
self.installProject = opsHelper.getValue( cfgPath( 'Pilot', 'Project' ), self.installProject )
self.installation = opsHelper.getValue( cfgPath( 'Pilot', 'Installation' ), self.installation )
self.pilotExtensionsList = opsHelper.getValue( "Pilot/Extensions", self.pilotExtensionsList )
self.log.info( '===============================================' )
self.log.info( 'Configuration:' )
self.log.info( '' )
self.log.info( ' Target Grids: ', ', '.join( self.targetGrids ) )
self.log.info( ' Install script: ', self.install )
self.log.info( ' Pilot script: ', self.pilot )
self.log.info( ' Install Ver: ', self.installVersion )
if self.installProject:
self.log.info( ' Project: ', self.installProject )
if self.installation:
self.log.info( ' Installation: ', self.installation )
if self.extraPilotOptions:
self.log.info( ' Extra Options: ', ' '.join( self.extraPilotOptions ) )
self.log.info( ' ListMatch: ', self.enableListMatch )
self.log.info( ' Private %: ', self.privatePilotFraction * 100 )
if self.enableListMatch:
self.log.info( ' ListMatch Delay:', self.listMatchDelay )
self.listMatchCache.purgeExpired()
def __getPrefix(self):
op = Operations("glast.org")
self.userprefix = None
res = getProxyInfo()
if res["OK"]:
if "username" in res["Value"]:
user = res["Value"]["username"]
self.userprefix = "/glast.org/user/%s/%s/" % (user[0], user)
else:
self.log.error("Proxy could not be found")
return 1
task_category = os.environ["GPL_TASKCATEGORY"]
if not task_category:
task_category = op.getValue("Pipeline/TaskCategory", None)
if not task_category:
self.log.error("Could not find task category")
return 1
self.prefixDest = op.getValue("Pipeline/StorageElementBasePath", self.userprefix)
self.stagingDest = (
self.prefixDest
+ "/"
+ task_category
+ "/"
+ os.environ["PIPELINE_TASK"]
+ "/"
+ os.environ["PIPELINE_STREAM"]
)
return 0
def configure(self, csSection, submitPool):
"""
Here goes common configuration for all PilotDirectors
"""
self.configureFromSection(csSection)
self.reloadConfiguration(csSection, submitPool)
# Get the defaults for the Setup where the Director is running
opsHelper = Operations()
self.installVersion = opsHelper.getValue(cfgPath("Pilot", "Version"), [self.installVersion])[0]
self.installProject = opsHelper.getValue(cfgPath("Pilot", "Project"), self.installProject)
self.installation = opsHelper.getValue(cfgPath("Pilot", "Installation"), self.installation)
self.pilotExtensionsList = opsHelper.getValue("Pilot/Extensions", self.pilotExtensionsList)
self.log.info("===============================================")
self.log.info("Configuration:")
self.log.info("")
self.log.info(" Target Grids: ", ", ".join(self.targetGrids))
self.log.info(" Install script: ", self.install)
self.log.info(" Pilot script: ", self.pilot)
self.log.info(" Pilot modules", self.extraModules)
self.log.info(" Install Ver: ", self.installVersion)
if self.installProject:
self.log.info(" Project: ", self.installProject)
if self.installation:
self.log.info(" Installation: ", self.installation)
if self.extraPilotOptions:
self.log.info(" Extra Options: ", " ".join(self.extraPilotOptions))
self.log.info(" ListMatch: ", self.enableListMatch)
self.log.info(" Private %: ", self.privatePilotFraction * 100)
if self.enableListMatch:
self.log.info(" ListMatch Delay:", self.listMatchDelay)
self.listMatchCache.purgeExpired()
def configure( self, csSection, submitPool ):
"""
Here goes common configuration for all PilotDirectors
"""
self.configureFromSection( csSection )
self.reloadConfiguration( csSection, submitPool )
# Get the defaults for the Setup where the Director is running
opsHelper = Operations()
self.installVersion = opsHelper.getValue( cfgPath( 'Pilot', 'Version' ), [ self.installVersion ] )[0]
self.installProject = opsHelper.getValue( cfgPath( 'Pilot', 'Project' ), self.installProject )
self.installation = opsHelper.getValue( cfgPath( 'Pilot', 'Installation' ), self.installation )
self.pilotExtensionsList = opsHelper.getValue( "Pilot/Extensions", self.pilotExtensionsList )
self.log.info( '===============================================' )
self.log.info( 'Configuration:' )
self.log.info( '' )
self.log.info( ' Target Grids: ', ', '.join( self.targetGrids ) )
self.log.info( ' Install script: ', self.install )
self.log.info( ' Pilot script: ', self.pilot )
self.log.info( ' Pilot modules', self.extraModules )
self.log.info( ' Install Ver: ', self.installVersion )
if self.installProject:
self.log.info( ' Project: ', self.installProject )
if self.installation:
self.log.info( ' Installation: ', self.installation )
if self.extraPilotOptions:
self.log.info( ' Extra Options: ', ' '.join( self.extraPilotOptions ) )
self.log.info( ' ListMatch: ', self.enableListMatch )
self.log.info( ' Private %: ', self.privatePilotFraction * 100 )
if self.enableListMatch:
self.log.info( ' ListMatch Delay:', self.listMatchDelay )
self.listMatchCache.purgeExpired()
def getPilotOutput(cls, pilotRef):
if not pilotRef.startswith("vm://"):
return S_ERROR("Invalid pilot reference %s" % pilotRef)
# Get the VM public IP
diracID, nPilot = os.path.basename(pilotRef).split(":")
result = cls.virtualMachineDB.getUniqueIDByName(diracID)
if not result["OK"]:
return result
uniqueID = result["Value"]
result = cls.virtualMachineDB.getInstanceID(uniqueID)
if not result["OK"]:
return result
instanceID = result["Value"]
result = cls.virtualMachineDB.getInstanceParameter("PublicIP", instanceID)
if not result["OK"]:
return result
publicIP = result["Value"]
op = Operations()
privateKeyFile = op.getValue("/Cloud/PrivateKey", "")
diracUser = op.getValue("/Cloud/VMUser", "")
ssh_str = "%s@%s" % (diracUser, publicIP)
cmd = ["ssh", "-i", privateKeyFile, ssh_str, "cat /etc/joboutputs/vm-pilot.%s.log" % nPilot]
inst = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE)
output, stderr = inst.communicate()
if inst.returncode:
return S_ERROR("Failed to get pilot output: %s" % stderr)
else:
return S_OK(output)
def __getCSOptions(self):
"""Get agent options from the CS."""
self.enabled = self.am_getOption('EnableFlag', False)
self.transformationsToIgnore = self.am_getOption(
'TransformationsToIgnore', [])
self.getJobInfoFromJDLOnly = self.am_getOption('JobInfoFromJDLOnly',
False)
self.transformationStatus = self.am_getOption('TransformationStatus',
['Active', 'Completing'])
ops = Operations()
extendableTTypes = set(
ops.getValue('Transformations/ExtendableTransfTypes',
['MCSimulation']))
dataProcessing = set(ops.getValue('Transformations/DataProcessing',
[]))
self.transNoInput = self.am_getOption('TransformationsNoInput',
list(extendableTTypes))
self.transWithInput = self.am_getOption(
'TransformationsWithInput',
list(dataProcessing - extendableTTypes))
self.transformationTypes = self.transWithInput + self.transNoInput
self.log.notice('Will treat transformations without input files',
self.transNoInput)
self.log.notice('Will treat transformations with input files',
self.transWithInput)
self.addressTo = self.am_getOption('MailTo', [])
self.addressFrom = self.am_getOption('MailFrom', '')
self.printEveryNJobs = self.am_getOption('PrintEvery', 200)
def initialize(self):
"""
Flags useESForJobParametersFlag (in /Operations/[]/Services/JobMonitoring/) have bool value (True/False)
and determines the switching of backends from MySQL to ElasticSearch for the JobParameters DB table.
For version v7r0, the MySQL backend is (still) the default.
"""
credDict = self.getRemoteCredentials()
self.ownerDN = credDict['DN']
self.ownerGroup = credDict['group']
operations = Operations(group=self.ownerGroup)
self.globalJobsInfo = operations.getValue(
'/Services/JobMonitoring/GlobalJobsInfo', True)
self.jobPolicy = JobPolicy(self.ownerDN, self.ownerGroup,
self.globalJobsInfo)
self.jobPolicy.jobDB = gJobDB
useESForJobParametersFlag = operations.getValue(
'/Services/JobMonitoring/useESForJobParametersFlag', False)
global gElasticJobDB
if useESForJobParametersFlag:
gElasticJobDB = ElasticJobDB()
self.log.verbose("Using ElasticSearch for JobParameters")
return S_OK()
def __getBackendsFromCFG(self, cfgPath):
"""
Get backends from the configuration and register them in LoggingRoot.
This is the new way to get the backends providing a general configuration.
:params cfgPath: string of the configuration path
"""
# We have to put the import line here to avoid a dependancy loop
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC import gConfig
# get the second last string representing the component type in the configuration
# example : 'Agents', 'Services'
component = cfgPath.split("/")[-2]
operation = Operations()
# Search desired backends in the component
desiredBackends = gConfig.getValue("%s/%s" % (cfgPath, 'LogBackends'), [])
if not desiredBackends:
# Search desired backends in the operation section according to the
# component type
desiredBackends = operation.getValue(
"Logging/Default%sBackends" % component, [])
if not desiredBackends:
# Search desired backends in the operation section
desiredBackends = operation.getValue("Logging/DefaultBackends", [])
if not desiredBackends:
# Default value
desiredBackends = ['stdout']
return desiredBackends
def check(self):
"""
Check that the manifest is OK
"""
for k in ["OwnerName", "OwnerDN", "OwnerGroup", "DIRACSetup"]:
if k not in self.__manifest:
return S_ERROR("Missing var %s in manifest" % k)
# Check CPUTime
result = self.__checkNumericalVar("CPUTime", 86400, 100, 500000)
if not result["OK"]:
return result
result = self.__checkNumericalVar("Priority", 1, 0, 10)
if not result["OK"]:
return result
maxInputData = Operations().getValue("JobDescription/MaxInputData",
500)
result = self.__checkMaxInputData(maxInputData)
if not result["OK"]:
return result
operation = Operations(group=self.__manifest["OwnerGroup"])
allowedJobTypes = operation.getValue("JobDescription/AllowedJobTypes",
["User", "Test", "Hospital"])
transformationTypes = operation.getValue(
"Transformations/DataProcessing", [])
result = self.__checkMultiChoice("JobType",
allowedJobTypes + transformationTypes)
if not result["OK"]:
return result
return S_OK()
def getTarBallLocation(app, config, dummy_area):
""" Get the tar ball location.
"""
ops = Operations()
appName = app[0]
appVersion = app[1]
appName = appName.lower()
app_tar = ops.getValue(
'/AvailableTarBalls/%s/%s/%s/TarBall' % (config, appName, appVersion),
'')
overwrite = ops.getValue(
'/AvailableTarBalls/%s/%s/%s/Overwrite' %
(config, appName, appVersion), False)
md5sum = ops.getValue(
'/AvailableTarBalls/%s/%s/%s/Md5Sum' % (config, appName, appVersion),
'')
gLogger.info("Looking for application %s%s for config %s:" %
(appName, appVersion, config))
if not app_tar:
gLogger.error('Could not find tar ball for %s %s' %
(appName, appVersion))
return S_ERROR('Could not find tar ball for %s %s' %
(appName, appVersion))
tarballURL = ops.getValue(
'/AvailableTarBalls/%s/%s/TarBallURL' % (config, appName), '')
if not tarballURL:
gLogger.error('Could not find tarballURL in CS for %s %s' %
(appName, appVersion))
return S_ERROR('Could not find tarballURL in CS')
return S_OK([app_tar, tarballURL, overwrite, md5sum])
def getPilotOutput(pilotRef):
if not pilotRef.startswith('vm://'):
return S_ERROR('Invalid pilot reference %s' % pilotRef)
# Get the VM public IP
diracID, nPilot = os.path.basename(pilotRef).split(':')
result = gVirtualMachineDB.getUniqueIDByName(diracID)
if not result['OK']:
return result
uniqueID = result['Value']
result = gVirtualMachineDB.getInstanceID(uniqueID)
if not result['OK']:
return result
instanceID = result['Value']
result = gVirtualMachineDB.getInstanceParameter("PublicIP", instanceID)
if not result['OK']:
return result
publicIP = result['Value']
op = Operations()
privateKeyFile = op.getValue('/Cloud/PrivateKey', '')
diracUser = op.getValue('/Cloud/VMUser', '')
cmd = 'ssh -i %s %s@%s "cat /etc/joboutputs/vm-pilot.%s.log"' % (
privateKeyFile, diracUser, publicIP, nPilot)
status, output = commands.getstatusoutput(cmd)
if status:
return S_ERROR('Failed to get pilot output: %s' % output)
else:
return S_OK(output)
def registerSwitchesAndParseCommandLine(self):
"""Register the default plus additional parameters and parse options.
:param list options: list of three tuple for options to add to the script
:param list flags: list of three tuple for flags to add to the script
:param str opName
"""
for short, longOption, doc in self.options:
Script.registerSwitch(short + ':' if short else '',
longOption + '=', doc)
for short, longOption, doc in self.flags:
Script.registerSwitch(short, longOption, doc)
self.switches[longOption] = False
Script.parseCommandLine()
if Script.getPositionalArgs():
Script.showHelp()
DIRAC.exit(1)
ops = Operations()
if not ops.getValue('DataManagement/ArchiveFiles/Enabled', False):
sLog.error(
'The "ArchiveFiles" operation is not enabled, contact your administrator!'
)
DIRAC.exit(1)
for _short, longOption, _doc in self.options:
defaultValue = ops.getValue(
'DataManagement/ArchiveFiles/%s' % longOption, None)
if defaultValue:
sLog.verbose(
'Found default value in the CS for %r with value %r' %
(longOption, defaultValue))
self.switches[longOption] = defaultValue
for _short, longOption, _doc in self.flags:
defaultValue = ops.getValue(
'DataManagement/ArchiveFiles/%s' % longOption, False)
if defaultValue:
sLog.verbose(
'Found default value in the CS for %r with value %r' %
(longOption, defaultValue))
self.switches[longOption] = defaultValue
for switch in Script.getUnprocessedSwitches():
for short, longOption, doc in self.options:
if switch[0] == short or switch[0].lower() == longOption.lower(
):
sLog.verbose('Found switch %r with value %r' %
(longOption, switch[1]))
self.switches[longOption] = switch[1]
break
for short, longOption, doc in self.flags:
if switch[0] == short or switch[0].lower() == longOption.lower(
):
self.switches[longOption] = True
break
self.checkSwitches()
self.switches['DryRun'] = not self.switches.get('Execute', False)
self.switches['SourceSE'] = self.switches.get('SourceSE',
'').split(',')
class RssConfiguration:
'''
RssConfiguration:
{
Config:
{
State : Active | InActive,
Cache : 300,
FromAddress : '*****@*****.**'
StatusType :
{
default : all,
StorageElement: ReadAccess, WriteAccess, CheckAccess, RemoveAccess
}
}
}
'''
def __init__(self):
self.opsHelper = Operations()
def getConfigCache(self, default=300):
'''
Gets from <pathToRSSConfiguration>/Config the value of Cache
'''
return self.opsHelper.getValue('%s/Config/Cache' % _rssConfigPath,
default)
def getConfigFromAddress(self, default=None):
'''
Gets from <pathToRSSConfiguration>/Config the value of FromAddress
'''
return self.opsHelper.getValue(
'%s/Config/FromAddress' % _rssConfigPath, default)
def getConfigStatusType(self, elementType=None):
'''
Gets all the status types per elementType, if not given, it takes default
from CS. If not, hardcoded variable DEFAULT.
'''
_DEFAULTS = ('all', )
res = self.opsHelper.getOptionsDict('%s/Config/StatusTypes' %
_rssConfigPath)
if res['OK']:
if elementType in res['Value']:
return List.fromChar(res['Value'][elementType])
if 'default' in res['Value']:
return List.fromChar(res['Value']['default'])
return _DEFAULTS
class RssConfiguration:
'''
RssConfiguration:
{
Config:
{
State : Active | InActive,
Cache : 300,
FromAddress : '*****@*****.**'
StatusType :
{
default : all,
StorageElement: ReadAccess, WriteAccess, CheckAccess, RemoveAccess
}
}
}
'''
def __init__( self ):
self.opsHelper = Operations()
def getConfigCache( self, default = 300 ):
'''
Gets from <pathToRSSConfiguration>/Config the value of Cache
'''
return self.opsHelper.getValue( '%s/Config/Cache' % _rssConfigPath, default )
def getConfigFromAddress( self, default = None ):
'''
Gets from <pathToRSSConfiguration>/Config the value of FromAddress
'''
return self.opsHelper.getValue( '%s/Config/FromAddress' % _rssConfigPath, default )
def getConfigStatusType( self, elementType = None ):
'''
Gets all the status types per elementType, if not given, it takes default
from CS. If not, hardcoded variable DEFAULT.
'''
_DEFAULTS = ( 'all', )
res = self.opsHelper.getOptionsDict( '%s/Config/StatusTypes' % _rssConfigPath )
if res[ 'OK' ]:
if elementType in res[ 'Value' ]:
return List.fromChar( res[ 'Value' ][ elementType ] )
if 'default' in res[ 'Value' ]:
return List.fromChar( res[ 'Value' ][ 'default' ] )
return _DEFAULTS
def getShifterProxy(shifterType, fileName=False):
"""
This method returns a shifter's proxy
:param shifterType: ProductionManager / DataManager...
"""
if fileName:
try:
os.makedirs(os.path.dirname(fileName))
except OSError:
pass
opsHelper = Operations()
userName = opsHelper.getValue(cfgPath('Shifter', shifterType, 'User'), '')
if not userName:
return S_ERROR("No shifter User defined for %s" % shifterType)
result = CS.getDNForUsername(userName)
if not result['OK']:
return result
userDN = result['Value'][0]
result = CS.findDefaultGroupForDN(userDN)
if not result['OK']:
return result
defaultGroup = result['Value']
userGroup = opsHelper.getValue(cfgPath('Shifter', shifterType, 'Group'),
defaultGroup)
vomsAttr = CS.getVOMSAttributeForGroup(userGroup)
if vomsAttr:
gLogger.info("Getting VOMS [%s] proxy for shifter %s@%s (%s)" %
(vomsAttr, userName, userGroup, userDN))
result = gProxyManager.downloadVOMSProxyToFile(userDN,
userGroup,
filePath=fileName,
requiredTimeLeft=86400,
cacheTime=86400)
else:
gLogger.info("Getting proxy for shifter %s@%s (%s)" %
(userName, userGroup, userDN))
result = gProxyManager.downloadProxyToFile(userDN,
userGroup,
filePath=fileName,
requiredTimeLeft=86400,
cacheTime=86400)
if not result['OK']:
return result
chain = result['chain']
fileName = result['Value']
return S_OK({
'DN': userDN,
'username': userName,
'group': userGroup,
'chain': chain,
'proxyFile': fileName
})
def sync(self):
""" Main synchronizer method.
"""
ops = Operations()
self._checksumDict = {}
self.pilotFileServer = ops.getValue("Pilot/pilotFileServer",
self.pilotFileServer)
if not self.pilotFileServer:
self.log.fatal(
"The /Operations/<Setup>/Pilot/pilotFileServer option is not defined"
)
self.log.fatal(
"Pilot 3 files won't be updated, and you won't be able to send pilots"
)
return S_OK(
"The /Operations/<Setup>/Pilot/pilotFileServer option is not defined"
)
self.workDir = ops.getValue("Pilot/workDir", self.workDir)
if self.workDir:
try:
os.mkdir(self.workDir)
except OSError:
pass
self.log.notice(
'-- Synchronizing the content of the JSON file with the content of the CS --',
'(%s)' % self.jsonFile)
self.pilotRepo = ops.getValue("Pilot/pilotRepo", self.pilotRepo)
self.pilotVORepo = ops.getValue("Pilot/pilotVORepo", self.pilotVORepo)
self.projectDir = ops.getValue("Pilot/projectDir", self.projectDir)
self.pilotScriptPath = ops.getValue("Pilot/pilotScriptsPath",
self.pilotScriptPath)
self.pilotVOScriptPath = ops.getValue("Pilot/pilotVOScriptsPath",
self.pilotVOScriptPath)
self.pilotRepoBranch = ops.getValue("Pilot/pilotRepoBranch",
self.pilotRepoBranch)
self.pilotVORepoBranch = ops.getValue("Pilot/pilotVORepoBranch",
self.pilotVORepoBranch)
self.uploadToWebApp = ops.getValue("Pilot/uploadToWebApp", True)
res = self._syncJSONFile()
if not res['OK']:
return res
self.log.notice(
'-- Synchronizing the pilot scripts with the content of the repository --',
'(%s)' % self.pilotRepo)
self._syncScripts()
self._syncChecksum()
return S_OK()
def getShifterProxy(shifterType, fileName=False):
"""This method returns a shifter's proxy
:param str shifterType: ProductionManager / DataManager...
:param str fileName: file name
:return: S_OK(dict)/S_ERROR()
"""
if fileName:
mkDir(os.path.dirname(fileName))
opsHelper = Operations()
userName = opsHelper.getValue(cfgPath("Shifter", shifterType, "User"), "")
if not userName:
return S_ERROR("No shifter User defined for %s" % shifterType)
result = Registry.getDNForUsername(userName)
if not result["OK"]:
return result
userDN = result["Value"][0]
result = Registry.findDefaultGroupForDN(userDN)
if not result["OK"]:
return result
defaultGroup = result["Value"]
userGroup = opsHelper.getValue(cfgPath("Shifter", shifterType, "Group"),
defaultGroup)
vomsAttr = Registry.getVOMSAttributeForGroup(userGroup)
if vomsAttr:
gLogger.info("Getting VOMS [%s] proxy for shifter %s@%s (%s)" %
(vomsAttr, userName, userGroup, userDN))
result = gProxyManager.downloadVOMSProxyToFile(userDN,
userGroup,
filePath=fileName,
requiredTimeLeft=86400,
cacheTime=86400)
else:
gLogger.info("Getting proxy for shifter %s@%s (%s)" %
(userName, userGroup, userDN))
result = gProxyManager.downloadProxyToFile(userDN,
userGroup,
filePath=fileName,
requiredTimeLeft=86400,
cacheTime=86400)
if not result["OK"]:
return result
chain = result["chain"]
fileName = result["Value"]
return S_OK({
"DN": userDN,
"username": userName,
"group": userGroup,
"chain": chain,
"proxyFile": fileName
})
def __init__(self, argumentsDict):
""" Standard constructor
"""
self.arguments = argumentsDict
self.name = COMPONENT_NAME
self.log = gLogger.getSubLogger(self.name)
op = Operations()
self.arguments.setdefault('Configuration', {})['AllReplicas'] = op.getValue('InputDataPolicy/AllReplicas', False)
self.arguments['Configuration'].setdefault('Protocol', op.getValue('InputDataPolicy/Protocols/Local', []))
self.arguments['Configuration'].setdefault('RemoteProtocol', op.getValue('InputDataPolicy/Protocols/Remote', []))
# By default put input data into the current directory
self.arguments.setdefault('InputDataDirectory', 'CWD')
def sync(self):
""" Main synchronizer method.
"""
ops = Operations()
self.pilotFileServer = ops.getValue("Pilot/pilotFileServer", self.pilotFileServer)
if not self.pilotFileServer:
self.log.fatal("The /Operations/<Setup>/Pilot/pilotFileServer option is not defined")
self.log.fatal("Pilot 3 files won't be updated, and you won't be able to send pilots")
return S_OK("The /Operations/<Setup>/Pilot/pilotFileServer option is not defined")
self.log.notice('-- Synchronizing the content of the JSON file with the content of the CS --',
'(%s)' % self.jsonFile)
self.pilotRepo = ops.getValue("Pilot/pilotRepo", self.pilotRepo)
self.pilotVORepo = ops.getValue("Pilot/pilotVORepo", self.pilotVORepo)
self.projectDir = ops.getValue("Pilot/projectDir", self.projectDir)
self.pilotScriptPath = ops.getValue("Pilot/pilotScriptsPath", self.pilotScriptPath)
self.pilotVOScriptPath = ops.getValue("Pilot/pilotVOScriptsPath", self.pilotVOScriptPath)
self.pilotRepoBranch = ops.getValue("Pilot/pilotRepoBranch", self.pilotRepoBranch)
self.pilotVORepoBranch = ops.getValue("Pilot/pilotVORepoBranch", self.pilotVORepoBranch)
self._syncJSONFile()
self.log.notice('-- Synchronizing the pilot scripts with the content of the repository --',
'(%s)' % self.pilotRepo)
self._syncScripts()
return S_OK()
def getShifterProxy( shifterType, fileName = False ):
"""
This method returns a shifter's proxy
:param shifterType: ProductionManager / DataManager...
"""
if fileName:
try:
os.makedirs( os.path.dirname( fileName ) )
except OSError:
pass
opsHelper = Operations()
userName = opsHelper.getValue( cfgPath( 'Shifter', shifterType, 'User' ), '' )
if not userName:
return S_ERROR( "No shifter User defined for %s" % shifterType )
result = CS.getDNForUsername( userName )
if not result[ 'OK' ]:
return result
userDN = result[ 'Value' ][0]
result = CS.findDefaultGroupForDN( userDN )
if not result['OK']:
return result
defaultGroup = result['Value']
userGroup = opsHelper.getValue( cfgPath( 'Shifter', shifterType, 'Group' ), defaultGroup )
vomsAttr = CS.getVOMSAttributeForGroup( userGroup )
if vomsAttr:
gLogger.info( "Getting VOMS [%s] proxy for shifter %s@%s (%s)" % ( vomsAttr, userName,
userGroup, userDN ) )
result = gProxyManager.downloadVOMSProxyToFile( userDN, userGroup,
filePath = fileName,
requiredTimeLeft = 86400,
cacheTime = 86400 )
else:
gLogger.info( "Getting proxy for shifter %s@%s (%s)" % ( userName, userGroup, userDN ) )
result = gProxyManager.downloadProxyToFile( userDN, userGroup,
filePath = fileName,
requiredTimeLeft = 86400,
cacheTime = 86400 )
if not result[ 'OK' ]:
return result
chain = result[ 'chain' ]
fileName = result[ 'Value' ]
return S_OK( { 'DN' : userDN,
'username' : userName,
'group' : userGroup,
'chain' : chain,
'proxyFile' : fileName } )
def initialize(self):
"""
Flags gESFlag and gMySQLFlag have bool values (True/False)
derived from dirac.cfg configuration file
Determines the switching of ElasticSearch and MySQL backends
"""
global gElasticJobDB, gJobDB
gESFlag = self.srv_getCSOption('useES', False)
if gESFlag:
gElasticJobDB = ElasticJobDB()
gMySQLFlag = self.srv_getCSOption('useMySQL', True)
if not gMySQLFlag:
gJobDB = False
credDict = self.getRemoteCredentials()
self.ownerDN = credDict['DN']
self.ownerGroup = credDict['group']
operations = Operations(group=self.ownerGroup)
self.globalJobsInfo = operations.getValue('/Services/JobMonitoring/GlobalJobsInfo', True)
self.jobPolicy = JobPolicy(self.ownerDN, self.ownerGroup, self.globalJobsInfo)
self.jobPolicy.jobDB = gJobDB
return S_OK()
def getValidStatusTypes():
'''
Returns from the OperationsHelper: RSSConfiguration/GeneralConfig/Resources
'''
DEFAULTS = {
'Site' : { 'StatusType' : "''" },
'Service' : { 'StatusType' : "''" },
'Resource' : { 'StatusType' : "''" },
'StorageElement': { 'StatusType' : [ 'Read', 'Write', 'Remove', 'Check' ] }
}
opHelper = Operations()
sections = opHelper.getSections( 'RSSConfiguration/GeneralConfig/Resources' )
if not sections[ 'OK' ]:
return DEFAULTS
result = {}
for section in sections[ 'Value' ]:
res = opHelper.getValue( 'RSSConfiguration/GeneralConfig/Resources/%s/StatusType' % section )
if res is None:
if DEFAULTS.has_key( section ):
result[ section ] = { 'StatusType' : DEFAULTS[ section ] }
else:
result[ section ] = { 'StatusType' : None }
else:
result[ section ] = { 'StatusType' : Utils.getTypedList( res ) }
return result
def _getSEList( self, SEType = 'ProductionOutputs', DataType = 'SimtelProd' ):
""" get from CS the list of available SE for data upload
"""
opsHelper = Operations()
optionName = os.path.join( SEType, DataType )
SEList = opsHelper.getValue( optionName , [] )
SEList = List.randomize( SEList )
DIRAC.gLogger.notice( 'List of %s SE: %s ' % ( SEType, SEList ) )
# # Check if the local SE is in the list. If yes try it first by reversing list order
localSEList = []
res = getSEsForSite( DIRAC.siteName() )
if res['OK']:
localSEList = res['Value']
retainedlocalSEList = []
for localSE in localSEList:
if localSE in SEList:
DIRAC.gLogger.notice( 'The local Storage Element is an available SE: ', localSE )
retainedlocalSEList.append( localSE )
SEList.remove( localSE )
SEList = retainedlocalSEList + SEList
if len( SEList ) == 0:
return DIRAC.S_ERROR( 'Error in building SEList' )
return DIRAC.S_OK( SEList )
def resolveDeps(sysconfig, appli, appversion):
""" Resolve the dependencies
:param str sysconfig: system configuration
:param str appli: application name
:param str appversion: application version
:return: list of dictionaries
"""
log = gLogger.getSubLogger("resolveDeps")
ops = Operations()
deps = ops.getSections('/AvailableTarBalls/%s/%s/%s/Dependencies' % (sysconfig, appli,
appversion), '')
depsarray = []
if deps['OK']:
for dep in deps['Value']:
vers = ops.getValue('/AvailableTarBalls/%s/%s/%s/Dependencies/%s/version' % (sysconfig, appli,
appversion, dep), '')
depvers = ''
if vers:
depvers = vers
else:
log.error("Retrieving dependency version for %s failed, skipping to next !" % (dep))
continue
log.verbose("Found dependency %s %s" % (dep, depvers))
depdict = {}
depdict["app"] = dep
depdict["version"] = depvers
depsarray.append(depdict)
##resolve recursive dependencies
depsofdeps = resolveDeps(sysconfig, dep, depvers)
depsarray.extend(depsofdeps)
else:
log.verbose("Could not find any dependency for %s %s, ignoring" % (appli, appversion))
return depsarray
def submitProbeJobs(self, ce):
""" Submit some jobs to the CEs
"""
#need credentials, should be there since the initialize
from DIRAC.Interfaces.API.Dirac import Dirac
d = Dirac()
from DIRAC.Interfaces.API.Job import Job
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
import os
ops = Operations("glast.org")
scriptname = ops.getValue("ResourceStatus/SofwareManagementScript",
self.script)
j = Job()
j.setDestinationCE(ce)
j.setCPUTime(1000)
j.setName("Probe %s" % ce)
j.setJobGroup("SoftwareProbe")
j.setExecutable("%s/GlastDIRAC/ResourceStatusSystem/Client/%s" %
(os.environ['DIRAC'], scriptname),
logFile='SoftwareProbe.log')
j.setOutputSandbox('*.log')
res = d.submit(j)
if not res['OK']:
return res
return S_OK()
def submitProbeJobs(self, ce):
""" Submit some jobs to the CEs
"""
#need credentials, should be there since the initialize
from DIRAC.Interfaces.API.Dirac import Dirac
d = Dirac()
from DIRAC.Interfaces.API.Job import Job
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
import DIRAC
ops = Operations()
scriptname = ops.getValue("ResourceStatus/SofwareManagementScript", self.script)
j = Job()
j.setDestinationCE(ce)
j.setCPUTime(1000)
j.setName("Probe %s" % ce)
j.setJobGroup("SoftwareProbe")
j.setExecutable("%s/GlastDIRAC/ResourceStatusSystem/Client/%s" % (DIRAC.rootPath, scriptname),
logFile='SoftwareProbe.log')
j.setOutputSandbox('*.log')
res = d.submit(j)
if not res['OK']:
return res
return S_OK()
def initializeLesHouchesFileManagerHandler(serviceInfo):
""" Initialize the service
"""
ops = Operations()
res = ops.getOptionsDict("/Models")
if not res["OK"]:
return res
templates = res["Value"]
cfgPath = serviceInfo["serviceSectionPath"]
location = ""
location = ops.getValue("%s/BasePath" % cfgPath, location)
if not location:
gLogger.error("Path to LesHouches files not defined")
return S_ERROR("Path to LesHouches files not defined in CS")
missing = False
global ModelsDict
for template, tfile in templates.items():
ModelsDict[template] = {}
ModelsDict[template]["file"] = tfile
if not tfile:
ModelsDict[template]["content"] = [""]
continue
file_path = os.path.join([location, tfile])
if not os.path.exists(file_path):
gLogger.error("Missing %s" % file_path)
missing = True
break
LesHouchesFile = open(file_path, "r")
ModelsDict[template]["content"] = LesHouchesFile.readlines()
LesHouchesFile.close()
if missing:
return S_ERROR("File missing")
return S_OK()
def __init__( self, **kwargs ):
Client.__init__( self, **kwargs )
opsH = Operations()
self.maxResetCounter = opsH.getValue( 'Productions/ProductionFilesMaxResetCounter', 10 )
self.setServer( 'Transformation/TransformationManager' )
def __init__(self, argumentsDict):
"""Standard constructor"""
self.arguments = argumentsDict
self.name = COMPONENT_NAME
self.log = gLogger.getSubLogger(self.name)
op = Operations()
self.arguments.setdefault("Configuration",
{})["AllReplicas"] = op.getValue(
"InputDataPolicy/AllReplicas", False)
self.arguments["Configuration"].setdefault(
"Protocol", op.getValue("InputDataPolicy/Protocols/Local", []))
self.arguments["Configuration"].setdefault(
"RemoteProtocol",
op.getValue("InputDataPolicy/Protocols/Remote", []))
# By default put input data into the current directory
self.arguments.setdefault("InputDataDirectory", "CWD")
def __getCSOptions(self):
"""Get agent options from the CS."""
self.enabled = self.am_getOption("EnableFlag", False)
self.transformationsToIgnore = self.am_getOption("TransformationsToIgnore", [])
self.getJobInfoFromJDLOnly = self.am_getOption("JobInfoFromJDLOnly", False)
self.transformationStatus = self.am_getOption("TransformationStatus", ["Active", "Completing"])
ops = Operations()
extendableTTypes = set(ops.getValue("Transformations/ExtendableTransfTypes", ["MCSimulation"]))
dataProcessing = set(ops.getValue("Transformations/DataProcessing", []))
self.transNoInput = self.am_getOption("TransformationsNoInput", list(extendableTTypes))
self.transWithInput = self.am_getOption("TransformationsWithInput", list(dataProcessing - extendableTTypes))
self.transformationTypes = self.transWithInput + self.transNoInput
self.log.notice("Will treat transformations without input files", self.transNoInput)
self.log.notice("Will treat transformations with input files", self.transWithInput)
self.addressTo = self.am_getOption("MailTo", [])
self.addressFrom = self.am_getOption("MailFrom", "")
self.printEveryNJobs = self.am_getOption("PrintEvery", 200)
def __init__(self, **kwargs):
Client.__init__(self, **kwargs)
opsH = Operations()
self.maxResetCounter = opsH.getValue(
'Productions/ProductionFilesMaxResetCounter', 10)
self.setServer('Transformation/TransformationManager')
def __getOwnerGroupDN(self, shifterType):
opsHelper = Operations()
userName = opsHelper.getValue(
cfgPath('BoincShifter', shifterType, 'User'), '')
if not userName:
return S_ERROR("No shifter User defined for %s" % shifterType)
result = CS.getDNForUsername(userName)
if not result['OK']:
return result
userDN = result['Value'][0]
result = CS.findDefaultGroupForDN(userDN)
if not result['OK']:
return result
defaultGroup = result['Value']
userGroup = opsHelper.getValue(
cfgPath('BoincShifter', shifterType, 'Group'), defaultGroup)
return userDN, userGroup, userName
def __init__(self):
"""c'tor
Just setting defaults
"""
self.workDir = "" # Working directory where the files are going to be stored
# domain name of the web server(s) used to upload the pilot json file and the pilot scripts
self.pilotFileServer = ""
# pilot sync default parameters
self.pilotRepo = "https://github.com/DIRACGrid/Pilot.git" # repository of the pilot
self.pilotVORepo = "" # repository of the VO that can contain a pilot extension
self.pilotSetup = gConfig.getValue("/DIRAC/Setup", "")
self.projectDir = ""
# where the find the pilot scripts in the VO pilot repository
self.pilotScriptPath = "Pilot" # where the find the pilot scripts in the pilot repository
self.pilotVOScriptPath = ""
self.pilotRepoBranch = "master"
self.pilotVORepoBranch = "master"
self.log = gLogger.getSubLogger(__name__)
ops = Operations()
# Overriding parameters from the CS
self.pilotRepo = ops.getValue("Pilot/pilotRepo", self.pilotRepo)
self.pilotVORepo = ops.getValue("Pilot/pilotVORepo", self.pilotVORepo)
self.projectDir = ops.getValue("Pilot/projectDir", self.projectDir)
self.pilotScriptPath = ops.getValue("Pilot/pilotScriptsPath", self.pilotScriptPath)
self.pilotVOScriptPath = ops.getValue("Pilot/pilotVOScriptsPath", self.pilotVOScriptPath)
self.pilotRepoBranch = ops.getValue("Pilot/pilotRepoBranch", self.pilotRepoBranch)
self.pilotVORepoBranch = ops.getValue("Pilot/pilotVORepoBranch", self.pilotVORepoBranch)
def __init__(self, argumentsDict):
""" Standard constructor
"""
self.arguments = argumentsDict
self.name = COMPONENT_NAME
self.log = gLogger.getSubLogger(self.name)
op = Operations()
self.arguments.setdefault('Configuration',
{})['AllReplicas'] = op.getValue(
'InputDataPolicy/AllReplicas', False)
self.arguments['Configuration'].setdefault(
'Protocol', op.getValue('InputDataPolicy/Protocols/Local', []))
self.arguments['Configuration'].setdefault(
'RemoteProtocol',
op.getValue('InputDataPolicy/Protocols/Remote', []))
# By default put input data into the current directory
self.arguments.setdefault('InputDataDirectory', 'CWD')
def __init__(self, **kwargs):
"""Simple constructor"""
super(TransformationClient, self).__init__(**kwargs)
opsH = Operations()
self.maxResetCounter = opsH.getValue(
"Transformations/FilesMaxResetCounter", 10)
self.setServer("Transformation/TransformationManager")
def allowedBkg( bkg, energy = None, detector = None, detectormodel = None, machine = 'clic_cdr' ):
""" Check is supplied bkg is allowed
"""
#gLogger.info("Those are the arguments: %s, %s, %s, %s, %s" % (bkg, energy, detector, detectormodel, machine) )
ops = Operations()
#bkg_allowed = [ 'gghad', 'pairs' ]
#if not bkg in bkg_allowed:
# return S_ERROR( "Bkg not allowed" )
res = -1
if energy:
if detectormodel:
res = ops.getValue( "/Overlay/%s/%s/%s/%s/ProdID" % (machine, energy, detectormodel, bkg), -1 )
if res < 0:
return S_ERROR( "No background to overlay" )
if detector:##needed for backward compatibility
res = ops.getValue( "/Overlay/%s/%s/%s/%s/ProdID" % (machine, detector, energy, bkg), -1 )
if res < 0 :
return S_ERROR( "No background to overlay" )
return S_OK(res)
def allowedBkg( bkg, energy = None, detector = None, detectormodel = None, machine = 'clic_cdr' ):
""" Check is supplied bkg is allowed
"""
#gLogger.info("Those are the arguments: %s, %s, %s, %s, %s" % (bkg, energy, detector, detectormodel, machine) )
ops = Operations()
#bkg_allowed = [ 'gghad', 'pairs' ]
#if not bkg in bkg_allowed:
# return S_ERROR( "Bkg not allowed" )
res = -1
if energy:
if detectormodel:
res = ops.getValue( "/Overlay/%s/%s/%s/%s/ProdID" % (machine, energy, detectormodel, bkg), -1 )
if res < 0:
return S_ERROR( "No background to overlay" )
if detector:##needed for backward compatibility
res = ops.getValue( "/Overlay/%s/%s/%s/%s/ProdID" % (machine, detector, energy, bkg), -1 )
if res < 0 :
return S_ERROR( "No background to overlay" )
return S_OK(res)
def __init__(self, **kwargs):
""" Simple constructor
"""
Client.__init__(self, **kwargs)
opsH = Operations()
self.maxResetCounter = opsH.getValue(
'Transformations/FilesMaxResetCounter', 10)
self.setServer('Transformation/TransformationManager')
def getCountryMappingTier1( country ):
""" Returns the Tier1 site mapped to a country code """
opsHelper = Operations()
res = getCountryMapping( country )
if not res['OK']:
return res
mappedCountry = res['Value']
tier1 = opsHelper.getValue( '/Countries/%s/Tier1' % mappedCountry, '' )
if not tier1:
return S_ERROR( "No Tier1 assigned to %s" % mappedCountry )
return S_OK( tier1 )
def getCountryMappingTier1(country):
""" Returns the Tier1 site mapped to a country code """
opsHelper = Operations()
res = getCountryMapping(country)
if not res['OK']:
return res
mappedCountry = res['Value']
tier1 = opsHelper.getValue('/Countries/%s/Tier1' % mappedCountry, '')
if not tier1:
return S_ERROR("No Tier1 assigned to %s" % mappedCountry)
return S_OK(tier1)
def constructUserLFNs(jobID, vo, owner, outputFiles, outputPath):
""" This method is used to supplant the standard job wrapper output data policy
for ILC. The initial convention adopted for user output files is the following:
If outputpath is not defined:
<vo>/user/<initial e.g. s>/<owner e.g. sposs>/<yearMonth e.g. 2010_02>/<subdir>/<fileName>
Otherwise:
<vo>/user/<initial e.g. s>/<owner e.g. sposs>/<outputPath>/<fileName>
"""
initial = owner[:1]
subdir = str(jobID/1000)
timeTup = datetime.date.today().timetuple()
yearMonth = '%s_%s' % (timeTup[0], string.zfill(str(timeTup[1]), 2))
outputLFNs = {}
if not vo:
#res = gConfig.getOption("/DIRAC/VirtualOrganization", "ilc")
res = getVOfromProxyGroup()
if not res['OK']:
gLogger.error('Could not get VO from CS, assuming ilc')
vo = 'ilc'
else:
vo = res['Value']
ops = Operations(vo = vo)
lfn_prefix = ops.getValue("LFNUserPrefix", "user")
#Strip out any leading or trailing slashes but allow fine structure
if outputPath:
outputPathList = string.split(outputPath, os.sep)
newPath = []
for i in outputPathList:
if i:
newPath.append(i)
outputPath = string.join(newPath, os.sep)
if not type(outputFiles) == types.ListType:
outputFiles = [outputFiles]
for outputFile in outputFiles:
#strip out any fine structure in the output file specified by the user, restrict to output file names
#the output path field can be used to describe this
outputFile = outputFile.replace('LFN:', '')
lfn = ''
if outputPath:
lfn = os.sep+os.path.join(vo, lfn_prefix, initial, owner, outputPath + os.sep + os.path.basename(outputFile))
else:
lfn = os.sep+os.path.join(vo, lfn_prefix, initial, owner, yearMonth, subdir, str(jobID)) + os.sep + os.path.basename(outputFile)
outputLFNs[outputFile] = lfn
outputData = outputLFNs.values()
if outputData:
gLogger.info('Created the following output data LFN(s):\n%s' % (string.join(outputData, '\n')))
else:
gLogger.info('No output LFN(s) constructed')
return S_OK(outputData)
def getSteeringFileDirName(platform, application, applicationVersion):
""" Locate the path of the steering file directory assigned to the specified application
"""
ops = Operations()
version = ops.getValue('/AvailableTarBalls/%s/%s/%s/Dependencies/steeringfiles/version' % (platform,
application,
applicationVersion), '')
if not version:
return S_ERROR("Could not find attached SteeringFile version")
return getSteeringFileDir(platform, version)
def initialize( self ):
credDict = self.getRemoteCredentials()
self.ownerDN = credDict['DN']
self.ownerGroup = credDict['group']
operations = Operations( group = self.ownerGroup )
self.globalJobsInfo = operations.getValue( '/Services/JobMonitoring/GlobalJobsInfo', True )
self.jobPolicy = JobPolicy( self.ownerDN, self.ownerGroup, self.globalJobsInfo )
self.jobPolicy.setJobDB( gJobDB )
return S_OK()
def initialize(self):
credDict = self.getRemoteCredentials()
self.ownerDN = credDict['DN']
self.ownerGroup = credDict['group']
operations = Operations(group=self.ownerGroup)
self.globalJobsInfo = operations.getValue('/Services/JobMonitoring/GlobalJobsInfo', True)
self.jobPolicy = JobPolicy(self.ownerDN, self.ownerGroup, self.globalJobsInfo)
self.jobPolicy.setJobDB(gJobDB)
return S_OK()
def getShifterProxy(shifterType, fileName=False):
"""
This method returns a shifter's proxy
- shifterType : ProductionManager / DataManager...
"""
if fileName:
try:
os.makedirs(os.path.dirname(fileName))
except OSError:
pass
opsHelper = Operations()
userName = opsHelper.getValue(cfgPath("Shifter", shifterType, "User"), "")
if not userName:
return S_ERROR("No shifter User defined for %s" % shifterType)
result = CS.getDNForUsername(userName)
if not result["OK"]:
return result
userDN = result["Value"][0]
result = CS.findDefaultGroupForDN(userDN)
if not result["OK"]:
return result
defaultGroup = result["Value"]
userGroup = opsHelper.getValue(cfgPath("Shifter", shifterType, "Group"), defaultGroup)
vomsAttr = CS.getVOMSAttributeForGroup(userGroup)
if vomsAttr:
gLogger.info("Getting VOMS [%s] proxy for shifter %s@%s (%s)" % (vomsAttr, userName, userGroup, userDN))
result = gProxyManager.downloadVOMSProxyToFile(
userDN, userGroup, filePath=fileName, requiredTimeLeft=1200, cacheTime=4 * 43200
)
else:
gLogger.info("Getting proxy for shifter %s@%s (%s)" % (userName, userGroup, userDN))
result = gProxyManager.downloadProxyToFile(
userDN, userGroup, filePath=fileName, requiredTimeLeft=1200, cacheTime=4 * 43200
)
if not result["OK"]:
return result
chain = result["chain"]
fileName = result["Value"]
return S_OK({"DN": userDN, "username": userName, "group": userGroup, "chain": chain, "proxyFile": fileName})
def getTarBallLocation(app, config, dummy_area):
""" Get the tar ball location.
"""
ops = Operations()
appName = app[0]
appVersion = app[1]
appName = appName.lower()
app_tar = ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall' % (config, appName, appVersion), '')
overwrite = ops.getValue('/AvailableTarBalls/%s/%s/%s/Overwrite' % (config, appName, appVersion), False)
md5sum = ops.getValue('/AvailableTarBalls/%s/%s/%s/Md5Sum' % (config, appName, appVersion), '')
gLogger.info("Looking for application %s%s for config %s:" % (appName, appVersion, config) )
if not app_tar:
gLogger.error('Could not find tar ball for %s %s'%(appName, appVersion))
return S_ERROR('Could not find tar ball for %s %s'%(appName, appVersion))
tarballURL = ops.getValue('/AvailableTarBalls/%s/%s/TarBallURL' % (config, appName), '')
if not tarballURL:
gLogger.error('Could not find tarballURL in CS for %s %s' % (appName, appVersion))
return S_ERROR('Could not find tarballURL in CS')
return S_OK([app_tar, tarballURL, overwrite, md5sum])
def getSteeringFileDirName(systemConfig, application, applicationVersion):
""" Locate the path of the steering file directory assigned to the specified application
"""
ops = Operations()
version = ops.getValue('/AvailableTarBalls/%s/%s/%s/Dependencies/steeringfiles/version' % (systemConfig,
application,
applicationVersion), '')
if not version:
return S_ERROR("Could not find attached SteeringFile version")
TarBall = ops.getValue('/AvailableTarBalls/%s/steeringfiles/%s/TarBall' % (systemConfig, version), '')
if not TarBall:
return S_ERROR("Could not find tar ball for SteeringFile")
mydir = TarBall.replace(".tgz", "").replace(".tar.gz", "")
res = getSoftwareFolder(mydir)
if not res['OK']:
return res
mySoftDir = res['Value']
res = check('steeringfiles.%s'%version,'.',[mySoftDir])##check that all the files are there: software is not corrupted.
if not res['OK']:
return res
return S_OK(mySoftDir)
def getSiteSEMapping(gridName=''):
""" Returns a dictionary of all sites and their localSEs as a list, e.g.
{'LCG.CERN.ch':['CERN-RAW','CERN-RDST',...]}
If gridName is specified, result is restricted to that Grid type.
"""
siteSEMapping = {}
gridTypes = gConfig.getSections('Resources/Sites/')
if not gridTypes['OK']:
gLogger.warn('Problem retrieving sections in /Resources/Sites')
return gridTypes
gridTypes = gridTypes['Value']
if gridName:
if not gridName in gridTypes:
return S_ERROR('Could not get sections for /Resources/Sites/%s' %
gridName)
gridTypes = [gridName]
gLogger.debug('Grid Types are: %s' % (', '.join(gridTypes)))
for grid in gridTypes:
sites = gConfig.getSections('/Resources/Sites/%s' % grid)
if not sites['OK']:
gLogger.warn('Problem retrieving /Resources/Sites/%s section' %
grid)
return sites
for candidate in sites['Value']:
candidateSEs = gConfig.getValue(
'/Resources/Sites/%s/%s/SE' % (grid, candidate), [])
if candidateSEs:
siteSEMapping[candidate] = candidateSEs
else:
gLogger.debug('No SEs defined for site %s' % candidate)
# Add Sites from the SiteLocalSEMapping in the CS
cfgLocalSEPath = cfgPath('SiteLocalSEMapping')
opsHelper = Operations()
result = opsHelper.getOptionsDict(cfgLocalSEPath)
if result['OK']:
mapping = result['Value']
for site in mapping:
ses = opsHelper.getValue(cfgPath(cfgLocalSEPath, site), [])
if not ses:
continue
if gridName:
if gridName != site.split('.')[0]:
continue
if site not in siteSEMapping:
siteSEMapping[site] = []
for se in ses:
if se not in siteSEMapping[site]:
siteSEMapping[site].append(se)
return S_OK(siteSEMapping)
def sync(self):
""" Main synchronizer method.
"""
ops = Operations()
self.pilotFileServer = ops.getValue("Pilot/pilotFileServer", self.pilotFileServer)
if not self.pilotFileServer:
gLogger.warn("The /Operations/<Setup>/Pilot/pilotFileServer option is not defined")
gLogger.warn("Pilot 3 files won't be updated, and you won't be able to use Pilot 3")
gLogger.warn("The Synchronization steps are anyway displayed")
gLogger.notice('-- Synchronizing the content of the JSON file %s with the content of the CS --' % self.jsonFile)
self.pilotRepo = ops.getValue("Pilot/pilotRepo", self.pilotRepo)
self.pilotVORepo = ops.getValue("Pilot/pilotVORepo", self.pilotVORepo)
self.projectDir = ops.getValue("Pilot/projectDir", self.projectDir)
self.pilotScriptPath = ops.getValue("Pilot/pilotScriptsPath", self.pilotScriptPath)
self.pilotVOScriptPath = ops.getValue("Pilot/pilotVOScriptsPath", self.pilotVOScriptPath)
result = self._syncJSONFile()
if not result['OK']:
gLogger.error("Error uploading the pilot file: %s" % result['Message'])
return result
gLogger.notice('-- Synchronizing the pilot scripts %s with the content of the repository --' % self.pilotRepo)
self._syncScripts()
return S_OK()
def getSiteSEMapping( gridName = '' ):
""" Returns a dictionary of all sites and their localSEs as a list, e.g.
{'LCG.CERN.ch':['CERN-RAW','CERN-RDST',...]}
If gridName is specified, result is restricted to that Grid type.
"""
siteSEMapping = {}
gridTypes = gConfig.getSections( 'Resources/Sites/' )
if not gridTypes['OK']:
gLogger.warn( 'Problem retrieving sections in /Resources/Sites' )
return gridTypes
gridTypes = gridTypes['Value']
if gridName:
if not gridName in gridTypes:
return S_ERROR( 'Could not get sections for /Resources/Sites/%s' % gridName )
gridTypes = [gridName]
gLogger.debug( 'Grid Types are: %s' % ( ', '.join( gridTypes ) ) )
for grid in gridTypes:
sites = gConfig.getSections( '/Resources/Sites/%s' % grid )
if not sites['OK']:
gLogger.warn( 'Problem retrieving /Resources/Sites/%s section' % grid )
return sites
for candidate in sites['Value']:
candidateSEs = gConfig.getValue( '/Resources/Sites/%s/%s/SE' % ( grid, candidate ), [] )
if candidateSEs:
siteSEMapping[candidate] = candidateSEs
else:
gLogger.debug( 'No SEs defined for site %s' % candidate )
# Add Sites from the SiteLocalSEMapping in the CS
cfgLocalSEPath = cfgPath( 'SiteLocalSEMapping' )
opsHelper = Operations()
result = opsHelper.getOptionsDict( cfgLocalSEPath )
if result['OK']:
mapping = result['Value']
for site in mapping:
ses = opsHelper.getValue( cfgPath( cfgLocalSEPath, site ), [] )
if not ses:
continue
if gridName:
if gridName != site.split( '.' )[0]:
continue
if site not in siteSEMapping:
siteSEMapping[site] = []
for se in ses:
if se not in siteSEMapping[site]:
siteSEMapping[site].append( se )
return S_OK( siteSEMapping )
def getCountryMapping( country ):
""" Determines the associated country from the country code"""
mappedCountries = [country]
opsHelper = Operations()
while True:
mappedCountry = opsHelper.getValue( '/Countries/%s/AssignedTo' % country, country )
if mappedCountry == country:
break
elif mappedCountry in mappedCountries:
return S_ERROR( 'Circular mapping detected for %s' % country )
else:
country = mappedCountry
mappedCountries.append( mappedCountry )
return S_OK( mappedCountry )
def getUserRootDir(self):
'''get user's initial root directory'''
username = self.getUserName()
initial = username[:1]
vo = getVO()
if not vo:
vo = 'bes'
ops = Operations(vo = vo)
user_prefix = ops.getValue('LFNUserPrefix', 'user')
basePath = '/' + vo + '/' + user_prefix + '/' + initial + '/' + username
return basePath
def resolveDepsTar(sysconfig, appli, appversion):
""" Return the dependency tar ball name, if available
Uses same parameters as L{resolveDeps}.
@return: array of strings
"""
ops = Operations()
deparray = resolveDeps(sysconfig, appli, appversion)
depsarray = []
for dep in deparray:
dep_tar = ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall' % (sysconfig, dep["app"],
dep["version"]), '')
if dep_tar:
depsarray.append(dep_tar)
else:
gLogger.error("Dependency %s version %s is not defined in CS, please check !" % (dep["app"], dep["version"]))
return depsarray
def __init__(self, dataDir):
op = Operations("glast.org")
self.stagingArea = dataDir
self.listFileStaged = []
self.nbofSEtried = 0
self.userprefix = None
self.prefixDest = None
self.stagingDest = None
SEtemporaryStaging = op.getValue("Pipeline/StorageElementTemporaryStaging", "")
self.listSEs = SEtemporaryStaging.split(",")
self.SE = None
self.log = gLogger.getSubLogger("GridAccess")
if self.__getPrefix():
self.log.error("Failed to initialize staging.")
raise Exception("Failed to initialize!")
self.__pickRandomSE()
def getSEsForCountry( country ):
""" Determines the associated SEs from the country code
"""
mappedCountries = [country]
opsHelper = Operations()
while True:
mappedCountry = opsHelper.getValue( '/Countries/%s/AssignedTo' % country, country )
if mappedCountry == country:
break
elif mappedCountry in mappedCountries:
return S_ERROR( 'Circular mapping detected for %s' % country )
else:
country = mappedCountry
mappedCountries.append( mappedCountry )
res = opsHelper.getOptionsDict( '/Countries/%s/AssociatedSEs' % country )
if not res['OK']:
return S_ERROR( 'Failed to obtain AssociatedSEs for %s' % country )
return S_OK( res['Value'].values() )
def _getProdLogs():
"""get production log files from LogSE"""
clip = _Params()
clip.registerSwitch()
Script.parseCommandLine()
if not ( clip.logF or clip.logD or clip.prodid ):
Script.showHelp()
dexit(1)
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
ops = Operations()
storageElementName = ops.getValue('/LogStorage/LogSE', 'LogSE')
from DIRAC.Resources.Storage.StorageElement import StorageElementItem as StorageElement
logSE = StorageElement(storageElementName)
if clip.prodid and not ( clip.logD or clip.logF ):
result = _getLogFolderFromID( clip )
if not result['OK']:
gLogger.error( result['Message'] )
dexit(1)
if clip.logD:
if not clip.noPromptBeforeDL:
res = promptUser('Are you sure you want to get ALL the files in this directory?')
if not res['OK']:
dexit()
choice = res['Value']
if choice.lower()=='n':
dexit(0)
if isinstance(clip.logD, str):
res = logSE.getDirectory(clip.logD, localPath=clip.outputdir)
_printErrorReport(res)
elif isinstance(clip.logD, list):
for logdir in clip.logD:
gLogger.notice('Getting log files from '+str(logdir))
res = logSE.getDirectory(logdir, localPath=clip.outputdir)
_printErrorReport(res)
if clip.logF:
res = logSE.getFile(clip.logF, localPath = clip.outputdir)
_printErrorReport(res)
class Matcher( object ):
""" Logic for matching
"""
def __init__( self, pilotAgentsDB = None, jobDB = None, tqDB = None, jlDB = None, opsHelper = None ):
""" c'tor
"""
if pilotAgentsDB:
self.pilotAgentsDB = pilotAgentsDB
else:
self.pilotAgentsDB = PilotAgentsDB()
if jobDB:
self.jobDB = jobDB
else:
self.jobDB = JobDB()
if tqDB:
self.tqDB = tqDB
else:
self.tqDB = TaskQueueDB()
if jlDB:
self.jlDB = jlDB
else:
self.jlDB = JobLoggingDB()
if opsHelper:
self.opsHelper = opsHelper
else:
self.opsHelper = Operations()
self.log = gLogger.getSubLogger( "Matcher" )
self.limiter = Limiter( jobDB = self.jobDB, opsHelper = self.opsHelper )
def selectJob( self, resourceDescription, credDict ):
""" Main job selection function to find the highest priority job matching the resource capacity
"""
startTime = time.time()
resourceDict = self._getResourceDict( resourceDescription, credDict )
negativeCond = self.limiter.getNegativeCondForSite( resourceDict['Site'] )
result = self.tqDB.matchAndGetJob( resourceDict, negativeCond = negativeCond )
if not result['OK']:
return result
result = result['Value']
if not result['matchFound']:
self.log.info( "No match found" )
raise RuntimeError( "No match found" )
jobID = result['jobId']
resAtt = self.jobDB.getJobAttributes( jobID, ['OwnerDN', 'OwnerGroup', 'Status'] )
if not resAtt['OK']:
raise RuntimeError( 'Could not retrieve job attributes' )
if not resAtt['Value']:
raise RuntimeError( "No attributes returned for job" )
if not resAtt['Value']['Status'] == 'Waiting':
self.log.error( 'Job matched by the TQ is not in Waiting state', str( jobID ) )
result = self.tqDB.deleteJob( jobID )
if not result[ 'OK' ]:
return result
raise RuntimeError( "Job %s is not in Waiting state" % str( jobID ) )
self._reportStatus( resourceDict, jobID )
result = self.jobDB.getJobJDL( jobID )
if not result['OK']:
raise RuntimeError( "Failed to get the job JDL" )
resultDict = {}
resultDict['JDL'] = result['Value']
resultDict['JobID'] = jobID
matchTime = time.time() - startTime
self.log.info( "Match time: [%s]" % str( matchTime ) )
gMonitor.addMark( "matchTime", matchTime )
# Get some extra stuff into the response returned
resOpt = self.jobDB.getJobOptParameters( jobID )
if resOpt['OK']:
for key, value in resOpt['Value'].items():
resultDict[key] = value
resAtt = self.jobDB.getJobAttributes( jobID, ['OwnerDN', 'OwnerGroup'] )
if not resAtt['OK']:
raise RuntimeError( 'Could not retrieve job attributes' )
if not resAtt['Value']:
raise RuntimeError( 'No attributes returned for job' )
if self.opsHelper.getValue( "JobScheduling/CheckMatchingDelay", True ):
self.limiter.updateDelayCounters( resourceDict['Site'], jobID )
pilotInfoReportedFlag = resourceDict.get( 'PilotInfoReportedFlag', False )
if not pilotInfoReportedFlag:
self._updatePilotInfo( resourceDict )
self._updatePilotJobMapping( resourceDict, jobID )
resultDict['DN'] = resAtt['Value']['OwnerDN']
resultDict['Group'] = resAtt['Value']['OwnerGroup']
resultDict['PilotInfoReportedFlag'] = True
return resultDict
def _getResourceDict( self, resourceDescription, credDict ):
""" from resourceDescription to resourceDict (just various mods)
"""
resourceDict = self._processResourceDescription( resourceDescription )
resourceDict = self._checkCredentials( resourceDict, credDict )
self._checkPilotVersion( resourceDict )
if not self._checkMask( resourceDict ):
# Banned destinations can only take Test jobs
resourceDict['JobType'] = 'Test'
self.log.verbose( "Resource description:" )
for key in resourceDict:
self.log.verbose( "%s : %s" % ( key.rjust( 20 ), resourceDict[ key ] ) )
return resourceDict
def _processResourceDescription( self, resourceDescription ):
""" Check and form the resource description dictionary
resourceDescription is a ceDict coming from a JobAgent, for example.
"""
resourceDict = {}
if isinstance( resourceDescription, basestring ):
classAdAgent = ClassAd( resourceDescription )
if not classAdAgent.isOK():
raise ValueError( 'Illegal Resource JDL' )
self.log.verbose( classAdAgent.asJDL() )
for name in singleValueDefFields:
if classAdAgent.lookupAttribute( name ):
if name == 'CPUTime':
resourceDict[name] = classAdAgent.getAttributeInt( name )
else:
resourceDict[name] = classAdAgent.getAttributeString( name )
for name in multiValueMatchFields:
if classAdAgent.lookupAttribute( name ):
if name == 'SubmitPool':
resourceDict[name] = classAdAgent.getListFromExpression( name )
else:
resourceDict[name] = classAdAgent.getAttributeString( name )
# Check if a JobID is requested
if classAdAgent.lookupAttribute( 'JobID' ):
resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' )
for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization' ):
if classAdAgent.lookupAttribute( k ):
resourceDict[ k ] = classAdAgent.getAttributeString( k )
else:
for name in singleValueDefFields:
if resourceDescription.has_key( name ):
resourceDict[name] = resourceDescription[name]
for name in multiValueMatchFields:
if resourceDescription.has_key( name ):
resourceDict[name] = resourceDescription[name]
if resourceDescription.has_key( 'JobID' ):
resourceDict['JobID'] = resourceDescription['JobID']
for k in ( 'DIRACVersion', 'ReleaseVersion', 'ReleaseProject', 'VirtualOrganization',
'PilotReference', 'PilotBenchmark', 'PilotInfoReportedFlag' ):
if k in resourceDescription:
resourceDict[ k ] = resourceDescription[ k ]
return resourceDict
def _reportStatus( self, resourceDict, jobID ):
""" Reports the status of the matched job in jobDB and jobLoggingDB
Do not fail if errors happen here
"""
attNames = ['Status', 'MinorStatus', 'ApplicationStatus', 'Site']
attValues = ['Matched', 'Assigned', 'Unknown', resourceDict['Site']]
result = self.jobDB.setJobAttributes( jobID, attNames, attValues )
if not result['OK']:
self.log.error( "Problem reporting job status", "setJobAttributes, jobID = %s: %s" % ( jobID, result['Message'] ) )
else:
self.log.verbose( "Set job attributes for jobID %s" % jobID )
result = self.jlDB.addLoggingRecord( jobID,
status = 'Matched',
minor = 'Assigned',
source = 'Matcher' )
if not result['OK']:
self.log.error( "Problem reporting job status", "addLoggingRecord, jobID = %s: %s" % ( jobID, result['Message'] ) )
else:
self.log.verbose( "Added logging record for jobID %s" % jobID )
def _checkMask( self, resourceDict ):
""" Check the mask: are we allowed to run normal jobs?
FIXME: should we move to site OR SE?
"""
if not 'Site' in resourceDict:
self.log.error( "Missing Site Name in Resource JDL" )
raise RuntimeError( "Missing Site Name in Resource JDL" )
# Get common site mask and check the agent site
result = self.jobDB.getSiteMask( siteState = 'Active' )
if not result['OK']:
self.log.error( "Internal error", "getSiteMask: %s" % result['Message'] )
raise RuntimeError( "Internal error" )
maskList = result['Value']
if resourceDict['Site'] not in maskList:
return False
return True
def _updatePilotInfo( self, resourceDict ):
""" Update pilot information - do not fail if we don't manage to do it
"""
pilotReference = resourceDict.get( 'PilotReference', '' )
if pilotReference:
gridCE = resourceDict.get( 'GridCE', 'Unknown' )
site = resourceDict.get( 'Site', 'Unknown' )
benchmark = resourceDict.get( 'PilotBenchmark', 0.0 )
self.log.verbose( 'Reporting pilot info for %s: gridCE=%s, site=%s, benchmark=%f' % ( pilotReference, gridCE, site, benchmark ) )
result = self.pilotAgentsDB.setPilotStatus( pilotReference, status = 'Running', gridSite = site,
destination = gridCE, benchmark = benchmark )
if not result['OK']:
self.log.error( "Problem updating pilot information",
"; setPilotStatus. pilotReference: %s; %s" % ( pilotReference, result['Message'] ) )
def _updatePilotJobMapping( self, resourceDict, jobID ):
""" Update pilot to job mapping information
"""
pilotReference = resourceDict.get( 'PilotReference', '' )
if pilotReference:
result = self.pilotAgentsDB.setCurrentJobID( pilotReference, jobID )
if not result['OK']:
self.log.error( "Problem updating pilot information",
";setCurrentJobID. pilotReference: %s; %s" % ( pilotReference, result['Message'] ) )
result = self.pilotAgentsDB.setJobForPilot( jobID, pilotReference, updateStatus = False )
if not result['OK']:
self.log.error( "Problem updating pilot information",
"; setJobForPilot. pilotReference: %s; %s" % ( pilotReference, result['Message'] ) )
def _checkCredentials( self, resourceDict, credDict ):
""" Check if we can get a job given the passed credentials
"""
if Properties.GENERIC_PILOT in credDict[ 'properties' ]:
# You can only match groups in the same VO
if credDict[ 'group' ] == "hosts":
# for the host case the VirtualOrganization parameter
# is mandatory in resourceDict
vo = resourceDict.get( 'VirtualOrganization', '' )
else:
vo = Registry.getVOForGroup( credDict[ 'group' ] )
result = Registry.getGroupsForVO( vo )
if result[ 'OK' ]:
resourceDict[ 'OwnerGroup' ] = result[ 'Value' ]
else:
raise RuntimeError( result['Message'] )
else:
# If it's a private pilot, the DN has to be the same
if Properties.PILOT in credDict[ 'properties' ]:
self.log.notice( "Setting the resource DN to the credentials DN" )
resourceDict[ 'OwnerDN' ] = credDict[ 'DN' ]
# If it's a job sharing. The group has to be the same and just check that the DN (if any)
# belongs to the same group
elif Properties.JOB_SHARING in credDict[ 'properties' ]:
resourceDict[ 'OwnerGroup' ] = credDict[ 'group' ]
self.log.notice( "Setting the resource group to the credentials group" )
if 'OwnerDN' in resourceDict and resourceDict[ 'OwnerDN' ] != credDict[ 'DN' ]:
ownerDN = resourceDict[ 'OwnerDN' ]
result = Registry.getGroupsForDN( resourceDict[ 'OwnerDN' ] )
if not result[ 'OK' ]:
raise RuntimeError( result['Message'] )
if credDict[ 'group' ] not in result[ 'Value' ]:
# DN is not in the same group! bad boy.
self.log.notice( "You cannot request jobs from DN %s. It does not belong to your group!" % ownerDN )
resourceDict[ 'OwnerDN' ] = credDict[ 'DN' ]
# Nothing special, group and DN have to be the same
else:
resourceDict[ 'OwnerDN' ] = credDict[ 'DN' ]
resourceDict[ 'OwnerGroup' ] = credDict[ 'group' ]
return resourceDict
def _checkPilotVersion( self, resourceDict ):
""" Check the pilot DIRAC version
"""
if self.opsHelper.getValue( "Pilot/CheckVersion", True ):
if 'ReleaseVersion' not in resourceDict:
if not 'DIRACVersion' in resourceDict:
raise RuntimeError( 'Version check requested and not provided by Pilot' )
else:
pilotVersion = resourceDict['DIRACVersion']
else:
pilotVersion = resourceDict['ReleaseVersion']
validVersions = self.opsHelper.getValue( "Pilot/Version", [] )
if validVersions and pilotVersion not in validVersions:
raise RuntimeError( 'Pilot version does not match the production version %s not in ( %s )' % \
( pilotVersion, ",".join( validVersions ) ) )
# Check project if requested
validProject = self.opsHelper.getValue( "Pilot/Project", "" )
if validProject:
if 'ReleaseProject' not in resourceDict:
raise RuntimeError( "Version check requested but expected project %s not received" % validProject )
if resourceDict[ 'ReleaseProject' ] != validProject:
raise RuntimeError( "Version check requested but expected project %s != received %s" % ( validProject,
resourceDict[ 'ReleaseProject' ] ) )
class JobManifest(object):
def __init__(self, manifest=""):
self.__manifest = CFG()
self.__dirty = False
self.__ops = False
if manifest:
result = self.load(manifest)
if not result["OK"]:
raise Exception(result["Message"])
def isDirty(self):
return self.__dirty
def setDirty(self):
self.__dirty = True
def clearDirty(self):
self.__dirty = False
def load(self, dataString):
"""
Auto discover format type based on [ .. ] of JDL
"""
dataString = dataString.strip()
if dataString[0] == "[" and dataString[-1] == "]":
return self.loadJDL(dataString)
else:
return self.loadCFG(dataString)
def loadJDL(self, jdlString):
"""
Load job manifest from JDL format
"""
result = loadJDLAsCFG(jdlString.strip())
if not result["OK"]:
self.__manifest = CFG()
return result
self.__manifest = result["Value"][0]
return S_OK()
def loadCFG(self, cfgString):
"""
Load job manifest from CFG format
"""
try:
self.__manifest.loadFromBuffer(cfgString)
except Exception as e:
return S_ERROR("Can't load manifest from cfg: %s" % str(e))
return S_OK()
def dumpAsCFG(self):
return str(self.__manifest)
def getAsCFG(self):
return self.__manifest.clone()
def dumpAsJDL(self):
return dumpCFGAsJDL(self.__manifest)
def __getCSValue(self, varName, defaultVal=None):
if not self.__ops:
self.__ops = Operations(group=self.__manifest["OwnerGroup"], setup=self.__manifest["DIRACSetup"])
if varName[0] != "/":
varName = "JobDescription/%s" % varName
return self.__ops.getValue(varName, defaultVal)
def __checkNumericalVar(self, varName, defaultVal, minVal, maxVal):
"""
Check a numerical var
"""
initialVal = False
if varName not in self.__manifest:
varValue = self.__getCSValue("Default%s" % varName, defaultVal)
else:
varValue = self.__manifest[varName]
initialVal = varValue
try:
varValue = long(varValue)
except:
return S_ERROR("%s must be a number" % varName)
minVal = self.__getCSValue("Min%s" % varName, minVal)
maxVal = self.__getCSValue("Max%s" % varName, maxVal)
varValue = max(minVal, min(varValue, maxVal))
if initialVal != varValue:
self.__manifest.setOption(varName, varValue)
return S_OK(varValue)
def __checkChoiceVar(self, varName, defaultVal, choices):
"""
Check a choice var
"""
initialVal = False
if varName not in self.__manifest:
varValue = self.__getCSValue("Default%s" % varName, defaultVal)
else:
varValue = self.__manifest[varName]
initialVal = varValue
if varValue not in self.__getCSValue("Choices%s" % varName, choices):
return S_ERROR("%s is not a valid value for %s" % (varValue, varName))
if initialVal != varValue:
self.__manifest.setOption(varName, varValue)
return S_OK(varValue)
def __checkMultiChoice(self, varName, choices):
"""
Check a multi choice var
"""
initialVal = False
if varName not in self.__manifest:
return S_OK()
else:
varValue = self.__manifest[varName]
initialVal = varValue
choices = self.__getCSValue("Choices%s" % varName, choices)
for v in List.fromChar(varValue):
if v not in choices:
return S_ERROR("%s is not a valid value for %s" % (v, varName))
if initialVal != varValue:
self.__manifest.setOption(varName, varValue)
return S_OK(varValue)
def __checkMaxInputData(self, maxNumber):
"""
Check Maximum Number of Input Data files allowed
"""
varName = "InputData"
if varName not in self.__manifest:
return S_OK()
varValue = self.__manifest[varName]
if len(List.fromChar(varValue)) > maxNumber:
return S_ERROR(
"Number of Input Data Files (%s) greater than current limit: %s"
% (len(List.fromChar(varValue)), maxNumber)
)
return S_OK()
def __contains__(self, key):
""" Check if the manifest has the required key
"""
return key in self.__manifest
def setOptionsFromDict(self, varDict):
for k in sorted(varDict):
self.setOption(k, varDict[k])
def check(self):
"""
Check that the manifest is OK
"""
for k in ["OwnerName", "OwnerDN", "OwnerGroup", "DIRACSetup"]:
if k not in self.__manifest:
return S_ERROR("Missing var %s in manifest" % k)
# Check CPUTime
result = self.__checkNumericalVar("CPUTime", 86400, 100, 500000)
if not result["OK"]:
return result
result = self.__checkNumericalVar("Priority", 1, 0, 10)
if not result["OK"]:
return result
allowedSubmitPools = getSubmitPools(self.__manifest["OwnerGroup"])
result = self.__checkMultiChoice("SubmitPools", list(set(allowedSubmitPools)))
if not result["OK"]:
return result
result = self.__checkMultiChoice("PilotTypes", ["private"])
if not result["OK"]:
return result
maxInputData = Operations().getValue("JobDescription/MaxInputData", 500)
result = self.__checkMaxInputData(maxInputData)
if not result["OK"]:
return result
transformationTypes = Operations().getValue("Transformations/DataProcessing", [])
result = self.__checkMultiChoice("JobType", ["User", "Test", "Hospital"] + transformationTypes)
if not result["OK"]:
return result
return S_OK()
def createSection(self, secName, contents=False):
if secName not in self.__manifest:
if contents and not isinstance(contents, CFG):
return S_ERROR("Contents for section %s is not a cfg object" % secName)
self.__dirty = True
return S_OK(self.__manifest.createNewSection(secName, contents=contents))
return S_ERROR("Section %s already exists" % secName)
def getSection(self, secName):
self.__dirty = True
sec = self.__manifest[secName]
if not sec:
return S_ERROR("%s does not exist")
return S_OK(sec)
def setSectionContents(self, secName, contents):
if contents and not isinstance(contents, CFG):
return S_ERROR("Contents for section %s is not a cfg object" % secName)
self.__dirty = True
if secName in self.__manifest:
self.__manifest[secName].reset()
self.__manifest[secName].mergeWith(contents)
else:
self.__manifest.createNewSection(secName, contents=contents)
def setOption(self, varName, varValue):
"""
Set a var in job manifest
"""
self.__dirty = True
levels = List.fromChar(varName, "/")
cfg = self.__manifest
for l in levels[:-1]:
if l not in cfg:
cfg.createNewSection(l)
cfg = cfg[l]
cfg.setOption(levels[-1], varValue)
def remove(self, opName):
levels = List.fromChar(opName, "/")
cfg = self.__manifest
for l in levels[:-1]:
if l not in cfg:
return S_ERROR("%s does not exist" % opName)
cfg = cfg[l]
if cfg.deleteKey(levels[-1]):
self.__dirty = True
return S_OK()
return S_ERROR("%s does not exist" % opName)
def getOption(self, varName, defaultValue=None):
"""
Get a variable from the job manifest
"""
cfg = self.__manifest
return cfg.getOption(varName, defaultValue)
def getOptionList(self, section=""):
"""
Get a list of variables in a section of the job manifest
"""
cfg = self.__manifest.getRecursive(section)
if not cfg or "value" not in cfg:
return []
cfg = cfg["value"]
return cfg.listOptions()
def isOption(self, opName):
"""
Check if it is a valid option
"""
return self.__manifest.isOption(opName)
def getSectionList(self, section=""):
"""
Get a list of sections in the job manifest
"""
cfg = self.__manifest.getRecursive(section)
if not cfg or "value" not in cfg:
return []
cfg = cfg["value"]
return cfg.listSections()
