def getControlledUsers(self, right):
"""Get users and groups which jobs are subject to the given access right"""
userGroupList = "ALL"
# If allInfo flag is defined we can see info for any job
if right == RIGHT_GET_INFO and self.allInfo:
return S_OK(userGroupList)
# Administrators can do everything
if Properties.JOB_ADMINISTRATOR in self.userProperties:
return S_OK(userGroupList)
# Inspectors can see info for all the jobs
if Properties.JOB_MONITOR in self.userProperties and right == RIGHT_GET_INFO:
return S_OK(userGroupList)
userGroupList = []
# User can do many things with his jobs
if Properties.NORMAL_USER in self.userProperties and right in OWNER_RIGHTS:
result = getGroupsForUser(self.userName)
if not result["OK"]:
return result
groups = result["Value"]
for group in groups:
if "NormalUser" in getPropertiesForGroup(group, []):
userGroupList.append((self.userName, group))
# User can do many things with the jobs in the shared group
if Properties.JOB_SHARING in self.userProperties and right in SHARED_GROUP_RIGHTS:
sharedUsers = getUsersInGroup(self.userGroup)
for user in sharedUsers:
userGroupList.append((user, self.userGroup))
userGroupList = list(set(userGroupList))
return S_OK(userGroupList)
def __init__(self, userDN, userGroup, allInfo=True):
self.userDN = userDN
self.userName = ''
result = getUsernameForDN(userDN)
if result['OK']:
self.userName = result['Value']
self.userGroup = userGroup
self.userProperties = getPropertiesForGroup(userGroup, [])
self.jobDB = None
self.allInfo = allInfo
self.__permissions = {}
self.__getUserJobPolicy()
def __init__( self, userDN, userGroup, allInfo=True ):
self.userDN = userDN
self.userName = ''
result = getUsernameForDN(userDN)
if result['OK']:
self.userName = result['Value']
self.userGroup = userGroup
self.userProperties = getPropertiesForGroup( userGroup, [] )
self.jobDB = None
self.allInfo = allInfo
self.__permissions = {}
self.__getUserJobPolicy()
def getControlledUsers( self, right ):
""" Get users and groups which jobs are subject to the given access right
"""
userGroupList = 'ALL'
# If allInfo flag is defined we can see info for any job
if right == RIGHT_GET_INFO and self.allInfo:
return S_OK( userGroupList )
# Administrators can do everything
if Properties.JOB_ADMINISTRATOR in self.userProperties:
return S_OK( userGroupList )
# Inspectors can see info for all the jobs
if Properties.JOB_MONITOR in self.userProperties and right == RIGHT_GET_INFO:
return S_OK( userGroupList )
userGroupList = []
# User can do many things with his jobs
if Properties.NORMAL_USER in self.userProperties and right in OWNER_RIGHTS:
result = getGroupsForUser( self.userName )
if not result['OK']:
return result
groups = result['Value']
for group in groups:
if 'NormalUser' in getPropertiesForGroup( group, [] ):
userGroupList.append( ( self.userName, group ) )
# User can do many things with the jobs in the shared group
if Properties.JOB_SHARING in self.userProperties and right in SHARED_GROUP_RIGHTS:
sharedUsers = getUsersInGroup( self.userGroup )
for user in sharedUsers:
userGroupList.append( ( user, self.userGroup ) )
userGroupList = list( set( userGroupList ) )
return S_OK( userGroupList )
def main():
Script.registerSwitch(
"C", "country",
"Sort site names by country postfix (i.e. LCG.IHEP.cn, LCG.IN2P3.fr, LCG.IHEP.su)",
sortBy)
Script.registerSwitch("R", "reverse", "Reverse the sort order", isReverse)
# Registering arguments will automatically add their description to the help menu
Script.registerArgument([
"Section: Name of the subsection in '/Resources/Sites/' for sort (i.e. LCG DIRAC)"
],
mandatory=False)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
result = getProxyInfo()
if not result["OK"]:
gLogger.error("Failed to get proxy information", result["Message"])
DIRACExit(2)
proxy = result["Value"]
if proxy["secondsLeft"] < 1:
gLogger.error("Your proxy has expired, please create new one")
DIRACExit(2)
group = proxy["group"]
if "CSAdministrator" not in getPropertiesForGroup(group):
gLogger.error(
"You must be CSAdministrator user to execute this script")
gLogger.notice(
"Please issue 'dirac-proxy-init -g [group with CSAdministrator Property]'"
)
DIRACExit(2)
cs = CSAPI()
result = cs.getCurrentCFG()
if not result["OK"]:
gLogger.error("Failed to get copy of CS", result["Message"])
DIRACExit(2)
cfg = result["Value"]
if not cfg.isSection("Resources"):
gLogger.error("Section '/Resources' is absent in CS")
DIRACExit(2)
if not cfg.isSection("Resources/Sites"):
gLogger.error("Subsection '/Resources/Sites' is absent in CS")
DIRACExit(2)
if args and len(args) > 0:
resultList = args[:]
else:
resultList = cfg["Resources"]["Sites"].listSections()
hasRun = False
isDirty = False
for i in resultList:
if not cfg.isSection("Resources/Sites/%s" % i):
gLogger.error("Subsection /Resources/Sites/%s does not exists" % i)
continue
hasRun = True
if SORTBYNAME:
dirty = cfg["Resources"]["Sites"][i].sortAlphabetically(
ascending=not REVERSE)
else:
dirty = cfg["Resources"]["Sites"][i].sortByKey(key=country,
reverse=REVERSE)
if dirty:
isDirty = True
if not hasRun:
gLogger.notice(
"Failed to find suitable subsections with site names to sort")
DIRACExit(0)
if not isDirty:
gLogger.notice("Nothing to do, site names are already sorted")
DIRACExit(0)
timestamp = toString(dateTime())
stamp = "Site names are sorted by %s script at %s" % (Script.scriptName,
timestamp)
cs.setOptionComment("/Resources/Sites", stamp)
result = cs.commit()
if not result["OK"]:
gLogger.error("Failed to commit changes to CS", result["Message"])
DIRACExit(2)
gLogger.notice("Site names are sorted and committed to CS")
DIRACExit(0)
def hasAdminAccess(self,credDict):
group = credDict.get('group','')
if FC_MANAGEMENT in getPropertiesForGroup(group):
return S_OK(True)
return S_OK(False)
def hasAdminAccess(self, credDict):
group = credDict.get('group', '')
if FC_MANAGEMENT in getPropertiesForGroup(group):
return S_OK(True)
return S_OK(False)
gridSiteName = args[1]
ces = args[2:]
try:
diracGridType, place, country = diracSiteName.split( '.' )
except:
gLogger.error( "The DIRACSiteName should be of the form GRID.LOCATION.COUNTRY for example LCG.CERN.ch" )
DIRAC.exit( -1 )
res = getProxyInfo()
if not res['OK']:
gLogger.error( "Failed to get proxy information", res['Message'] )
DIRAC.exit( 2 )
userName = res['Value']['username']
group = res['Value']['group']
if not 'CSAdministrator' in getPropertiesForGroup( group ):
gLogger.error( "You must be CSAdministrator user to execute this script" )
gLogger.notice( "Please issue 'dirac-proxy-init -g [group with CSAdministrator Property]'" )
DIRAC.exit( 2 )
cfgBase = "/Resources/Sites/%s/%s" % ( diracGridType, diracSiteName )
res = gConfig.getOptionsDict( cfgBase )
if res['OK'] and res['Value']:
gLogger.error( "The site %s is already defined:" % diracSiteName )
for key, value in res['Value'].items():
gLogger.notice( "%s = %s" % ( key, value ) )
DIRAC.exit( 2 )
csAPI.setOption( "%s/Name" % cfgBase, gridSiteName )
csAPI.setOption( "%s/CE" % cfgBase, ','.join( ces ) )
res = csAPI.commitChanges()
)
)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
result = getProxyInfo()
if not result["OK"]:
gLogger.error("Failed to get proxy information", result["Message"])
DIRAC.exit(2)
proxy = result["Value"]
if proxy["secondsLeft"] < 1:
gLogger.error("Your proxy has expired, please create new one")
DIRAC.exit(2)
group = proxy["group"]
if not "CSAdministrator" in getPropertiesForGroup(group):
gLogger.error("You must be CSAdministrator user to execute this script")
gLogger.notice("Please issue 'dirac-proxy-init -g [group with CSAdministrator Property]'")
DIRAC.exit(2)
cs = CSAPI()
result = cs.getCurrentCFG()
if not result["OK"]:
gLogger.error("Failed to get copy of CS", result["Message"])
DIRAC.exit(2)
cfg = result["Value"]
if not cfg.isSection("Resources"):
gLogger.error("Section '/Resources' is absent in CS")
DIRAC.exit(2)
def _getPilotOptions( self, taskQueueDict, pilotsToSubmit ):
# Need to limit the maximum number of pilots to submit at once
# For generic pilots this is limited by the number of use of the tokens and the
# maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation:
pilotsToSubmit = min( pilotsToSubmit, int( 50 / self.maxJobsInFillMode ) )
pilotOptions = []
privateIfGenericTQ = self.privatePilotFraction > random.random()
privateTQ = ( 'PilotTypes' in taskQueueDict and 'private' in [ t.lower() for t in taskQueueDict['PilotTypes'] ] )
forceGeneric = 'ForceGeneric' in taskQueueDict
submitPrivatePilot = ( privateIfGenericTQ or privateTQ ) and not forceGeneric
if submitPrivatePilot:
self.log.verbose( 'Submitting private pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] )
ownerDN = taskQueueDict['OwnerDN']
ownerGroup = taskQueueDict['OwnerGroup']
# User Group requirement
pilotOptions.append( '-G %s' % taskQueueDict['OwnerGroup'] )
# check if group allows jobsharing
ownerGroupProperties = getPropertiesForGroup( ownerGroup )
if not 'JobSharing' in ownerGroupProperties:
# Add Owner requirement to pilot
pilotOptions.append( "-O '%s'" % ownerDN )
if privateTQ:
pilotOptions.append( '-o /Resources/Computing/CEDefaults/PilotType=private' )
maxJobsInFillMode = self.maxJobsInFillMode
else:
#For generic jobs we'll submit mixture of generic and private pilots
self.log.verbose( 'Submitting generic pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] )
ownerDN = self.genericPilotDN
ownerGroup = self.genericPilotGroup
result = gProxyManager.requestToken( ownerDN, ownerGroup, max( pilotsToSubmit, self.maxJobsInFillMode ) )
if not result[ 'OK' ]:
self.log.error( ERROR_TOKEN, result['Message'] )
return S_ERROR( ERROR_TOKEN )
( token, numberOfUses ) = result[ 'Value' ]
pilotsToSubmit = min( numberOfUses, pilotsToSubmit )
pilotOptions.append( '-o /Security/ProxyToken=%s' % token )
pilotsToSubmit = ( pilotsToSubmit - 1 ) / self.maxJobsInFillMode + 1
maxJobsInFillMode = int( numberOfUses / pilotsToSubmit )
# Use Filling mode
pilotOptions.append( '-M %s' % maxJobsInFillMode )
# Debug
pilotOptions.append( '-d' )
# Setup.
pilotOptions.append( '-S %s' % taskQueueDict['Setup'] )
# CS Servers
csServers = gConfig.getValue( "/DIRAC/Configuration/Servers", [] )
pilotOptions.append( '-C %s' % ",".join( csServers ) )
# DIRAC Extensions
extensionsList = getCSExtensions()
if extensionsList:
pilotOptions.append( '-e %s' % ",".join( extensionsList ) )
# Requested version of DIRAC
pilotOptions.append( '-r %s' % self.installVersion )
# Requested Project to install
if self.installInstallation:
pilotOptions.append( '-V %s' % self.installInstallation )
# Requested CPU time
pilotOptions.append( '-T %s' % taskQueueDict['CPUTime'] )
if self.extraPilotOptions:
pilotOptions.extend( self.extraPilotOptions )
return S_OK( ( pilotOptions, pilotsToSubmit, ownerDN, ownerGroup, submitPrivatePilot, privateTQ ) )
gridSiteName = args[1]
ces = args[2:]
try:
diracGridType, place, country = diracSiteName.split( '.' )
except:
gLogger.error( "The DIRACSiteName should be of the form GRID.LOCATION.COUNTRY for example LCG.CERN.ch" )
DIRAC.exit( -1 )
res = getProxyInfo()
if not res['OK']:
gLogger.error( "Failed to get proxy information", res['Message'] )
DIRAC.exit( 2 )
userName = res['Value']['username']
group = res['Value']['group']
if not 'CSAdministrator' in getPropertiesForGroup( group ):
gLogger.error( "You must be CSAdministrator user to execute this script" )
gLogger.notice( "Please issue 'dirac-proxy-init -g [group with CSAdministrator Property]'" )
DIRAC.exit( 2 )
cfgBase = "/Resources/Sites/%s/%s" % ( diracGridType, diracSiteName )
res = gConfig.getOptionsDict( cfgBase )
if res['OK'] and res['Value']:
gLogger.error( "The site %s is already defined:" % diracSiteName )
for key, value in res['Value'].items():
gLogger.notice( "%s = %s" % ( key, value ) )
DIRAC.exit( 2 )
csAPI.setOption( "%s/Name" % cfgBase, gridSiteName )
csAPI.setOption( "%s/CE" % cfgBase, ','.join( ces ) )
res = csAPI.commitChanges()
def _getPilotOptions(self, taskQueueDict, pilotsToSubmit):
# Need to limit the maximum number of pilots to submit at once
# For generic pilots this is limited by the number of use of the tokens and the
# maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation:
pilotsToSubmit = max(min(pilotsToSubmit, int(50 / self.maxJobsInFillMode)), 1)
pilotOptions = []
privateIfGenericTQ = self.privatePilotFraction > random.random()
privateTQ = "PilotTypes" in taskQueueDict and "private" in [t.lower() for t in taskQueueDict["PilotTypes"]]
forceGeneric = "ForceGeneric" in taskQueueDict
submitPrivatePilot = (privateIfGenericTQ or privateTQ) and not forceGeneric
if submitPrivatePilot:
self.log.verbose("Submitting private pilots for TaskQueue %s" % taskQueueDict["TaskQueueID"])
ownerDN = taskQueueDict["OwnerDN"]
ownerGroup = taskQueueDict["OwnerGroup"]
# User Group requirement
pilotOptions.append("-G %s" % taskQueueDict["OwnerGroup"])
# check if group allows jobsharing
ownerGroupProperties = getPropertiesForGroup(ownerGroup)
if not "JobSharing" in ownerGroupProperties:
# Add Owner requirement to pilot
pilotOptions.append("-O '%s'" % ownerDN)
if privateTQ:
pilotOptions.append("-o /Resources/Computing/CEDefaults/PilotType=private")
maxJobsInFillMode = self.maxJobsInFillMode
else:
# For generic jobs we'll submit mixture of generic and private pilots
self.log.verbose("Submitting generic pilots for TaskQueue %s" % taskQueueDict["TaskQueueID"])
# ADRI: Find the generic group
result = findGenericPilotCredentials(group=taskQueueDict["OwnerGroup"])
if not result["OK"]:
self.log.error(ERROR_GENERIC_CREDENTIALS, result["Message"])
return S_ERROR(ERROR_GENERIC_CREDENTIALS)
ownerDN, ownerGroup = result["Value"]
result = gProxyManager.requestToken(ownerDN, ownerGroup, max(pilotsToSubmit, self.maxJobsInFillMode))
if not result["OK"]:
self.log.error(ERROR_TOKEN, result["Message"])
return S_ERROR(ERROR_TOKEN)
(token, numberOfUses) = result["Value"]
pilotsToSubmit = min(numberOfUses, pilotsToSubmit)
pilotOptions.append("-o /Security/ProxyToken=%s" % token)
pilotsToSubmit = max(1, (pilotsToSubmit - 1) / self.maxJobsInFillMode + 1)
maxJobsInFillMode = int(numberOfUses / pilotsToSubmit)
# Use Filling mode
pilotOptions.append("-M %s" % maxJobsInFillMode)
# Debug
pilotOptions.append("-d")
# Setup.
pilotOptions.append("-S %s" % taskQueueDict["Setup"])
# CS Servers
csServers = gConfig.getServersList()
if len(csServers) > 3:
# Remove the master
master = gConfigurationData.getMasterServer()
if master in csServers:
csServers.remove(master)
pilotOptions.append("-C %s" % ",".join(csServers))
# DIRAC Extensions
extensionsList = getCSExtensions()
if extensionsList:
pilotOptions.append("-e %s" % ",".join(extensionsList))
# Get DIRAC version and project, There might be global Setup defaults and per VO/Setup defaults (from configure)
opsHelper = Operations(group=taskQueueDict["OwnerGroup"], setup=taskQueueDict["Setup"])
# Requested version of DIRAC (it can be a list, so we take the fist one)
version = opsHelper.getValue(cfgPath("Pilot", "Version"), [self.installVersion])[0]
pilotOptions.append("-r %s" % version)
# Requested Project to install
installProject = opsHelper.getValue(cfgPath("Pilot", "Project"), self.installProject)
if installProject:
pilotOptions.append("-l %s" % installProject)
installation = opsHelper.getValue(cfgPath("Pilot", "Installation"), self.installation)
if installation:
pilotOptions.append("-V %s" % installation)
# Requested CPU time
pilotOptions.append("-T %s" % taskQueueDict["CPUTime"])
if self.submitPoolOption not in self.extraPilotOptions:
pilotOptions.append(self.submitPoolOption)
if self.extraPilotOptions:
pilotOptions.extend(self.extraPilotOptions)
return S_OK((pilotOptions, pilotsToSubmit, ownerDN, ownerGroup, submitPrivatePilot, privateTQ))
""
]))
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
result = getProxyInfo()
if not result["OK"]:
gLogger.error("Failed to get proxy information", result["Message"])
DIRAC.exit(2)
proxy = result["Value"]
if proxy["secondsLeft"] < 1:
gLogger.error("Your proxy has expired, please create new one")
DIRAC.exit(2)
group = proxy["group"]
if not "CSAdministrator" in getPropertiesForGroup(group):
gLogger.error("You must be CSAdministrator user to execute this script")
gLogger.notice(
"Please issue 'dirac-proxy-init -g [group with CSAdministrator Property]'"
)
DIRAC.exit(2)
cs = CSAPI()
result = cs.getCurrentCFG()
if not result["OK"]:
gLogger.error("Failed to get copy of CS", result["Message"])
DIRAC.exit(2)
cfg = result["Value"]
if not cfg.isSection("Resources"):
gLogger.error("Section '/Resources' is absent in CS")
def _getPilotOptions( self, taskQueueDict, pilotsToSubmit ):
# Need to limit the maximum number of pilots to submit at once
# For generic pilots this is limited by the number of use of the tokens and the
# maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation:
pilotsToSubmit = min( pilotsToSubmit, int( 50 / self.maxJobsInFillMode ) )
pilotOptions = [ "-V %s" % self.virtualOrganization ]
privateIfGenericTQ = self.privatePilotFraction > random.random()
privateTQ = ( 'PilotTypes' in taskQueueDict and 'private' in [ t.lower() for t in taskQueueDict['PilotTypes'] ] )
forceGeneric = 'ForceGeneric' in taskQueueDict
submitPrivatePilot = ( privateIfGenericTQ or privateTQ ) and not forceGeneric
if submitPrivatePilot:
self.log.verbose( 'Submitting private pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] )
ownerDN = taskQueueDict['OwnerDN']
ownerGroup = taskQueueDict['OwnerGroup']
# User Group requirement
pilotOptions.append( '-G %s' % taskQueueDict['OwnerGroup'] )
# check if group allows jobsharing
ownerGroupProperties = getPropertiesForGroup( ownerGroup )
if not 'JobSharing' in ownerGroupProperties:
# Add Owner requirement to pilot
pilotOptions.append( "-O '%s'" % ownerDN )
if privateTQ:
pilotOptions.append( '-o /Resources/Computing/CEDefaults/PilotType=private' )
maxJobsInFillMode = self.maxJobsInFillMode
else:
#For generic jobs we'll submit mixture of generic and private pilots
self.log.verbose( 'Submitting generic pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] )
ownerDN = self.genericPilotDN
ownerGroup = self.genericPilotGroup
result = gProxyManager.requestToken( ownerDN, ownerGroup, max( pilotsToSubmit, self.maxJobsInFillMode ) )
if not result[ 'OK' ]:
self.log.error( ERROR_TOKEN, result['Message'] )
return S_ERROR( ERROR_TOKEN )
( token, numberOfUses ) = result[ 'Value' ]
pilotsToSubmit = min( numberOfUses, pilotsToSubmit )
pilotOptions.append( '-o /Security/ProxyToken=%s' % token )
pilotsToSubmit = ( pilotsToSubmit - 1 ) / self.maxJobsInFillMode + 1
maxJobsInFillMode = int( numberOfUses / pilotsToSubmit )
# Use Filling mode
pilotOptions.append( '-M %s' % maxJobsInFillMode )
# Debug
pilotOptions.append( '-d' )
# Setup.
pilotOptions.append( '-S %s' % taskQueueDict['Setup'] )
# CS Servers
csServers = gConfig.getValue( "/DIRAC/Configuration/Servers", [] )
pilotOptions.append( '-C %s' % ",".join( csServers ) )
# DIRAC Extensions
extensionsList = getCSExtensions()
if extensionsList:
pilotOptions.append( '-e %s' % ",".join( extensionsList ) )
# Requested version of DIRAC
pilotOptions.append( '-r %s' % self.installVersion )
# Requested Project to install
pilotOptions.append( '-V %s' % self.installInstallation )
# Requested CPU time
pilotOptions.append( '-T %s' % taskQueueDict['CPUTime'] )
if self.extraPilotOptions:
pilotOptions.extend( self.extraPilotOptions )
return S_OK( ( pilotOptions, pilotsToSubmit, ownerDN, ownerGroup, submitPrivatePilot, privateTQ ) )
def _getPilotOptions( self, taskQueueDict, pilotsToSubmit ):
# Need to limit the maximum number of pilots to submit at once
# For generic pilots this is limited by the number of use of the tokens and the
# maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation:
pilotsToSubmit = max( min( pilotsToSubmit, int( 50 / self.maxJobsInFillMode ) ), 1 )
pilotOptions = []
privateIfGenericTQ = self.privatePilotFraction > random.random()
privateTQ = ( 'PilotTypes' in taskQueueDict and 'private' in [ t.lower() for t in taskQueueDict['PilotTypes'] ] )
forceGeneric = 'ForceGeneric' in taskQueueDict
submitPrivatePilot = ( privateIfGenericTQ or privateTQ ) and not forceGeneric
if submitPrivatePilot:
self.log.verbose( 'Submitting private pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] )
ownerDN = taskQueueDict['OwnerDN']
ownerGroup = taskQueueDict['OwnerGroup']
# User Group requirement
pilotOptions.append( '-G %s' % taskQueueDict['OwnerGroup'] )
# check if group allows jobsharing
ownerGroupProperties = getPropertiesForGroup( ownerGroup )
if not 'JobSharing' in ownerGroupProperties:
# Add Owner requirement to pilot
pilotOptions.append( "-O '%s'" % ownerDN )
if privateTQ:
pilotOptions.append( '-o /Resources/Computing/CEDefaults/PilotType=private' )
maxJobsInFillMode = self.maxJobsInFillMode
else:
#For generic jobs we'll submit mixture of generic and private pilots
self.log.verbose( 'Submitting generic pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] )
#ADRI: Find the generic group
result = findGenericPilotCredentials( group = taskQueueDict[ 'OwnerGroup' ] )
if not result[ 'OK' ]:
self.log.error( ERROR_GENERIC_CREDENTIALS, result[ 'Message' ] )
return S_ERROR( ERROR_GENERIC_CREDENTIALS )
ownerDN, ownerGroup = result[ 'Value' ]
result = gProxyManager.requestToken( ownerDN, ownerGroup, max( pilotsToSubmit, self.maxJobsInFillMode ) )
if not result[ 'OK' ]:
self.log.error( ERROR_TOKEN, result['Message'] )
return S_ERROR( ERROR_TOKEN )
( token, numberOfUses ) = result[ 'Value' ]
pilotsToSubmit = min( numberOfUses, pilotsToSubmit )
pilotOptions.append( '-o /Security/ProxyToken=%s' % token )
pilotsToSubmit = max( 1, ( pilotsToSubmit - 1 ) / self.maxJobsInFillMode + 1 )
maxJobsInFillMode = int( numberOfUses / pilotsToSubmit )
# Use Filling mode
pilotOptions.append( '-M %s' % maxJobsInFillMode )
# Debug
pilotOptions.append( '-d' )
# Setup.
pilotOptions.append( '-S %s' % taskQueueDict['Setup'] )
# CS Servers
csServers = gConfig.getServersList()
if len( csServers ) > 3:
# Remove the master
master = gConfigurationData.getMasterServer()
if master in csServers:
csServers.remove( master )
pilotOptions.append( '-C %s' % ",".join( csServers ) )
# DIRAC Extensions to be used in pilots
# ubeda: I'm not entirely sure if we can use here the same opsHelper as in line
# line +352
pilotExtensionsList = Operations().getValue( "Pilot/Extensions", [] )
extensionsList = []
if pilotExtensionsList:
if pilotExtensionsList[0] != 'None':
extensionsList = pilotExtensionsList
else:
extensionsList = getCSExtensions()
if extensionsList:
pilotOptions.append( '-e %s' % ",".join( extensionsList ) )
#Get DIRAC version and project, There might be global Setup defaults and per VO/Setup defaults (from configure)
opsHelper = Operations( group = taskQueueDict['OwnerGroup'], setup = taskQueueDict['Setup'] )
# Requested version of DIRAC (it can be a list, so we take the fist one)
version = opsHelper.getValue( cfgPath( 'Pilot', 'Version' ) , [ self.installVersion ] )[0]
pilotOptions.append( '-r %s' % version )
# Requested Project to install
installProject = opsHelper.getValue( cfgPath( 'Pilot', 'Project' ) , self.installProject )
if installProject:
pilotOptions.append( '-l %s' % installProject )
installation = opsHelper.getValue( cfgPath( 'Pilot', 'Installation' ), self.installation )
if installation:
pilotOptions.append( "-V %s" % installation )
# Requested CPU time
pilotOptions.append( '-T %s' % taskQueueDict['CPUTime'] )
if self.submitPoolOption not in self.extraPilotOptions:
pilotOptions.append( self.submitPoolOption )
if self.extraPilotOptions:
pilotOptions.extend( self.extraPilotOptions )
return S_OK( ( pilotOptions, pilotsToSubmit, ownerDN, ownerGroup, submitPrivatePilot, privateTQ ) )
def _getPilotOptions(self, taskQueueDict, pilotsToSubmit):
# Need to limit the maximum number of pilots to submit at once
# For generic pilots this is limited by the number of use of the tokens and the
# maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation:
pilotsToSubmit = max(
min(pilotsToSubmit, int(50 / self.maxJobsInFillMode)), 1)
pilotOptions = []
privateIfGenericTQ = self.privatePilotFraction > random.random()
privateTQ = ('PilotTypes' in taskQueueDict and 'private'
in [t.lower() for t in taskQueueDict['PilotTypes']])
forceGeneric = 'ForceGeneric' in taskQueueDict
submitPrivatePilot = (privateIfGenericTQ
or privateTQ) and not forceGeneric
if submitPrivatePilot:
self.log.verbose('Submitting private pilots for TaskQueue %s' %
taskQueueDict['TaskQueueID'])
ownerDN = taskQueueDict['OwnerDN']
ownerGroup = taskQueueDict['OwnerGroup']
# User Group requirement
pilotOptions.append('-G %s' % taskQueueDict['OwnerGroup'])
# check if group allows jobsharing
ownerGroupProperties = getPropertiesForGroup(ownerGroup)
if not 'JobSharing' in ownerGroupProperties:
# Add Owner requirement to pilot
pilotOptions.append("-O '%s'" % ownerDN)
if privateTQ:
pilotOptions.append(
'-o /Resources/Computing/CEDefaults/PilotType=private')
maxJobsInFillMode = self.maxJobsInFillMode
else:
#For generic jobs we'll submit mixture of generic and private pilots
self.log.verbose('Submitting generic pilots for TaskQueue %s' %
taskQueueDict['TaskQueueID'])
#ADRI: Find the generic group
result = findGenericPilotCredentials(
group=taskQueueDict['OwnerGroup'])
if not result['OK']:
self.log.error(ERROR_GENERIC_CREDENTIALS, result['Message'])
return S_ERROR(ERROR_GENERIC_CREDENTIALS)
ownerDN, ownerGroup = result['Value']
result = gProxyManager.requestToken(
ownerDN, ownerGroup, max(pilotsToSubmit,
self.maxJobsInFillMode))
if not result['OK']:
self.log.error(ERROR_TOKEN, result['Message'])
return S_ERROR(ERROR_TOKEN)
(token, numberOfUses) = result['Value']
pilotsToSubmit = min(numberOfUses, pilotsToSubmit)
pilotOptions.append('-o /Security/ProxyToken=%s' % token)
pilotsToSubmit = max(
1, (pilotsToSubmit - 1) / self.maxJobsInFillMode + 1)
maxJobsInFillMode = int(numberOfUses / pilotsToSubmit)
# Use Filling mode
pilotOptions.append('-M %s' % maxJobsInFillMode)
# Debug
pilotOptions.append('-d')
# Setup.
pilotOptions.append('-S %s' % taskQueueDict['Setup'])
# CS Servers
csServers = gConfig.getServersList()
if len(csServers) > 3:
# Remove the master
master = gConfigurationData.getMasterServer()
if master in csServers:
csServers.remove(master)
pilotOptions.append('-C %s' % ",".join(csServers))
# DIRAC Extensions to be used in pilots
# ubeda: I'm not entirely sure if we can use here the same opsHelper as in line
# line +352
pilotExtensionsList = Operations().getValue("Pilot/Extensions", [])
extensionsList = []
if pilotExtensionsList:
if pilotExtensionsList[0] != 'None':
extensionsList = pilotExtensionsList
else:
extensionsList = getCSExtensions()
if extensionsList:
pilotOptions.append('-e %s' % ",".join(extensionsList))
#Get DIRAC version and project, There might be global Setup defaults and per VO/Setup defaults (from configure)
opsHelper = Operations(group=taskQueueDict['OwnerGroup'],
setup=taskQueueDict['Setup'])
# Requested version of DIRAC (it can be a list, so we take the fist one)
version = opsHelper.getValue(cfgPath('Pilot', 'Version'),
[self.installVersion])[0]
pilotOptions.append('-r %s' % version)
# Requested Project to install
installProject = opsHelper.getValue(cfgPath('Pilot', 'Project'),
self.installProject)
if installProject:
pilotOptions.append('-l %s' % installProject)
installation = opsHelper.getValue(cfgPath('Pilot', 'Installation'),
self.installation)
if installation:
pilotOptions.append("-V %s" % installation)
# Requested CPU time
pilotOptions.append('-T %s' % taskQueueDict['CPUTime'])
if self.submitPoolOption not in self.extraPilotOptions:
pilotOptions.append(self.submitPoolOption)
if self.extraPilotOptions:
pilotOptions.extend(self.extraPilotOptions)
return S_OK((pilotOptions, pilotsToSubmit, ownerDN, ownerGroup,
submitPrivatePilot, privateTQ))
""
]))
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
result = getProxyInfo()
if not result["OK"]:
gLogger.error("Failed to get proxy information", result["Message"])
DIRACExit(2)
proxy = result["Value"]
if proxy["secondsLeft"] < 1:
gLogger.error("Your proxy has expired, please create new one")
DIRACExit(2)
group = proxy["group"]
if "CSAdministrator" not in getPropertiesForGroup(group):
gLogger.error("You must be CSAdministrator user to execute this script")
gLogger.notice(
"Please issue 'dirac-proxy-init -g [group with CSAdministrator Property]'"
)
DIRACExit(2)
cs = CSAPI()
result = cs.getCurrentCFG()
if not result["OK"]:
gLogger.error("Failed to get copy of CS", result["Message"])
DIRACExit(2)
cfg = result["Value"]
if not cfg.isSection("Resources"):
gLogger.error("Section '/Resources' is absent in CS")
