def getControlledUsers(self, right): """Get users and groups which jobs are subject to the given access right""" userGroupList = "ALL" # If allInfo flag is defined we can see info for any job if right == RIGHT_GET_INFO and self.allInfo: return S_OK(userGroupList) # Administrators can do everything if Properties.JOB_ADMINISTRATOR in self.userProperties: return S_OK(userGroupList) # Inspectors can see info for all the jobs if Properties.JOB_MONITOR in self.userProperties and right == RIGHT_GET_INFO: return S_OK(userGroupList) userGroupList = [] # User can do many things with his jobs if Properties.NORMAL_USER in self.userProperties and right in OWNER_RIGHTS: result = getGroupsForUser(self.userName) if not result["OK"]: return result groups = result["Value"] for group in groups: if "NormalUser" in getPropertiesForGroup(group, []): userGroupList.append((self.userName, group)) # User can do many things with the jobs in the shared group if Properties.JOB_SHARING in self.userProperties and right in SHARED_GROUP_RIGHTS: sharedUsers = getUsersInGroup(self.userGroup) for user in sharedUsers: userGroupList.append((user, self.userGroup)) userGroupList = list(set(userGroupList)) return S_OK(userGroupList)
def __init__(self, userDN, userGroup, allInfo=True): self.userDN = userDN self.userName = '' result = getUsernameForDN(userDN) if result['OK']: self.userName = result['Value'] self.userGroup = userGroup self.userProperties = getPropertiesForGroup(userGroup, []) self.jobDB = None self.allInfo = allInfo self.__permissions = {} self.__getUserJobPolicy()
def __init__( self, userDN, userGroup, allInfo=True ): self.userDN = userDN self.userName = '' result = getUsernameForDN(userDN) if result['OK']: self.userName = result['Value'] self.userGroup = userGroup self.userProperties = getPropertiesForGroup( userGroup, [] ) self.jobDB = None self.allInfo = allInfo self.__permissions = {} self.__getUserJobPolicy()
def getControlledUsers( self, right ): """ Get users and groups which jobs are subject to the given access right """ userGroupList = 'ALL' # If allInfo flag is defined we can see info for any job if right == RIGHT_GET_INFO and self.allInfo: return S_OK( userGroupList ) # Administrators can do everything if Properties.JOB_ADMINISTRATOR in self.userProperties: return S_OK( userGroupList ) # Inspectors can see info for all the jobs if Properties.JOB_MONITOR in self.userProperties and right == RIGHT_GET_INFO: return S_OK( userGroupList ) userGroupList = [] # User can do many things with his jobs if Properties.NORMAL_USER in self.userProperties and right in OWNER_RIGHTS: result = getGroupsForUser( self.userName ) if not result['OK']: return result groups = result['Value'] for group in groups: if 'NormalUser' in getPropertiesForGroup( group, [] ): userGroupList.append( ( self.userName, group ) ) # User can do many things with the jobs in the shared group if Properties.JOB_SHARING in self.userProperties and right in SHARED_GROUP_RIGHTS: sharedUsers = getUsersInGroup( self.userGroup ) for user in sharedUsers: userGroupList.append( ( user, self.userGroup ) ) userGroupList = list( set( userGroupList ) ) return S_OK( userGroupList )
def main(): Script.registerSwitch( "C", "country", "Sort site names by country postfix (i.e. LCG.IHEP.cn, LCG.IN2P3.fr, LCG.IHEP.su)", sortBy) Script.registerSwitch("R", "reverse", "Reverse the sort order", isReverse) # Registering arguments will automatically add their description to the help menu Script.registerArgument([ "Section: Name of the subsection in '/Resources/Sites/' for sort (i.e. LCG DIRAC)" ], mandatory=False) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() result = getProxyInfo() if not result["OK"]: gLogger.error("Failed to get proxy information", result["Message"]) DIRACExit(2) proxy = result["Value"] if proxy["secondsLeft"] < 1: gLogger.error("Your proxy has expired, please create new one") DIRACExit(2) group = proxy["group"] if "CSAdministrator" not in getPropertiesForGroup(group): gLogger.error( "You must be CSAdministrator user to execute this script") gLogger.notice( "Please issue 'dirac-proxy-init -g [group with CSAdministrator Property]'" ) DIRACExit(2) cs = CSAPI() result = cs.getCurrentCFG() if not result["OK"]: gLogger.error("Failed to get copy of CS", result["Message"]) DIRACExit(2) cfg = result["Value"] if not cfg.isSection("Resources"): gLogger.error("Section '/Resources' is absent in CS") DIRACExit(2) if not cfg.isSection("Resources/Sites"): gLogger.error("Subsection '/Resources/Sites' is absent in CS") DIRACExit(2) if args and len(args) > 0: resultList = args[:] else: resultList = cfg["Resources"]["Sites"].listSections() hasRun = False isDirty = False for i in resultList: if not cfg.isSection("Resources/Sites/%s" % i): gLogger.error("Subsection /Resources/Sites/%s does not exists" % i) continue hasRun = True if SORTBYNAME: dirty = cfg["Resources"]["Sites"][i].sortAlphabetically( ascending=not REVERSE) else: dirty = cfg["Resources"]["Sites"][i].sortByKey(key=country, reverse=REVERSE) if dirty: isDirty = True if not hasRun: gLogger.notice( "Failed to find suitable subsections with site names to sort") DIRACExit(0) if not isDirty: gLogger.notice("Nothing to do, site names are already sorted") DIRACExit(0) timestamp = toString(dateTime()) stamp = "Site names are sorted by %s script at %s" % (Script.scriptName, timestamp) cs.setOptionComment("/Resources/Sites", stamp) result = cs.commit() if not result["OK"]: gLogger.error("Failed to commit changes to CS", result["Message"]) DIRACExit(2) gLogger.notice("Site names are sorted and committed to CS") DIRACExit(0)
def hasAdminAccess(self,credDict): group = credDict.get('group','') if FC_MANAGEMENT in getPropertiesForGroup(group): return S_OK(True) return S_OK(False)
def hasAdminAccess(self, credDict): group = credDict.get('group', '') if FC_MANAGEMENT in getPropertiesForGroup(group): return S_OK(True) return S_OK(False)
gridSiteName = args[1] ces = args[2:] try: diracGridType, place, country = diracSiteName.split( '.' ) except: gLogger.error( "The DIRACSiteName should be of the form GRID.LOCATION.COUNTRY for example LCG.CERN.ch" ) DIRAC.exit( -1 ) res = getProxyInfo() if not res['OK']: gLogger.error( "Failed to get proxy information", res['Message'] ) DIRAC.exit( 2 ) userName = res['Value']['username'] group = res['Value']['group'] if not 'CSAdministrator' in getPropertiesForGroup( group ): gLogger.error( "You must be CSAdministrator user to execute this script" ) gLogger.notice( "Please issue 'dirac-proxy-init -g [group with CSAdministrator Property]'" ) DIRAC.exit( 2 ) cfgBase = "/Resources/Sites/%s/%s" % ( diracGridType, diracSiteName ) res = gConfig.getOptionsDict( cfgBase ) if res['OK'] and res['Value']: gLogger.error( "The site %s is already defined:" % diracSiteName ) for key, value in res['Value'].items(): gLogger.notice( "%s = %s" % ( key, value ) ) DIRAC.exit( 2 ) csAPI.setOption( "%s/Name" % cfgBase, gridSiteName ) csAPI.setOption( "%s/CE" % cfgBase, ','.join( ces ) ) res = csAPI.commitChanges()
) ) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() result = getProxyInfo() if not result["OK"]: gLogger.error("Failed to get proxy information", result["Message"]) DIRAC.exit(2) proxy = result["Value"] if proxy["secondsLeft"] < 1: gLogger.error("Your proxy has expired, please create new one") DIRAC.exit(2) group = proxy["group"] if not "CSAdministrator" in getPropertiesForGroup(group): gLogger.error("You must be CSAdministrator user to execute this script") gLogger.notice("Please issue 'dirac-proxy-init -g [group with CSAdministrator Property]'") DIRAC.exit(2) cs = CSAPI() result = cs.getCurrentCFG() if not result["OK"]: gLogger.error("Failed to get copy of CS", result["Message"]) DIRAC.exit(2) cfg = result["Value"] if not cfg.isSection("Resources"): gLogger.error("Section '/Resources' is absent in CS") DIRAC.exit(2)
def _getPilotOptions( self, taskQueueDict, pilotsToSubmit ): # Need to limit the maximum number of pilots to submit at once # For generic pilots this is limited by the number of use of the tokens and the # maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation: pilotsToSubmit = min( pilotsToSubmit, int( 50 / self.maxJobsInFillMode ) ) pilotOptions = [] privateIfGenericTQ = self.privatePilotFraction > random.random() privateTQ = ( 'PilotTypes' in taskQueueDict and 'private' in [ t.lower() for t in taskQueueDict['PilotTypes'] ] ) forceGeneric = 'ForceGeneric' in taskQueueDict submitPrivatePilot = ( privateIfGenericTQ or privateTQ ) and not forceGeneric if submitPrivatePilot: self.log.verbose( 'Submitting private pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] ) ownerDN = taskQueueDict['OwnerDN'] ownerGroup = taskQueueDict['OwnerGroup'] # User Group requirement pilotOptions.append( '-G %s' % taskQueueDict['OwnerGroup'] ) # check if group allows jobsharing ownerGroupProperties = getPropertiesForGroup( ownerGroup ) if not 'JobSharing' in ownerGroupProperties: # Add Owner requirement to pilot pilotOptions.append( "-O '%s'" % ownerDN ) if privateTQ: pilotOptions.append( '-o /Resources/Computing/CEDefaults/PilotType=private' ) maxJobsInFillMode = self.maxJobsInFillMode else: #For generic jobs we'll submit mixture of generic and private pilots self.log.verbose( 'Submitting generic pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] ) ownerDN = self.genericPilotDN ownerGroup = self.genericPilotGroup result = gProxyManager.requestToken( ownerDN, ownerGroup, max( pilotsToSubmit, self.maxJobsInFillMode ) ) if not result[ 'OK' ]: self.log.error( ERROR_TOKEN, result['Message'] ) return S_ERROR( ERROR_TOKEN ) ( token, numberOfUses ) = result[ 'Value' ] pilotsToSubmit = min( numberOfUses, pilotsToSubmit ) pilotOptions.append( '-o /Security/ProxyToken=%s' % token ) pilotsToSubmit = ( pilotsToSubmit - 1 ) / self.maxJobsInFillMode + 1 maxJobsInFillMode = int( numberOfUses / pilotsToSubmit ) # Use Filling mode pilotOptions.append( '-M %s' % maxJobsInFillMode ) # Debug pilotOptions.append( '-d' ) # Setup. pilotOptions.append( '-S %s' % taskQueueDict['Setup'] ) # CS Servers csServers = gConfig.getValue( "/DIRAC/Configuration/Servers", [] ) pilotOptions.append( '-C %s' % ",".join( csServers ) ) # DIRAC Extensions extensionsList = getCSExtensions() if extensionsList: pilotOptions.append( '-e %s' % ",".join( extensionsList ) ) # Requested version of DIRAC pilotOptions.append( '-r %s' % self.installVersion ) # Requested Project to install if self.installInstallation: pilotOptions.append( '-V %s' % self.installInstallation ) # Requested CPU time pilotOptions.append( '-T %s' % taskQueueDict['CPUTime'] ) if self.extraPilotOptions: pilotOptions.extend( self.extraPilotOptions ) return S_OK( ( pilotOptions, pilotsToSubmit, ownerDN, ownerGroup, submitPrivatePilot, privateTQ ) )
def _getPilotOptions(self, taskQueueDict, pilotsToSubmit): # Need to limit the maximum number of pilots to submit at once # For generic pilots this is limited by the number of use of the tokens and the # maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation: pilotsToSubmit = max(min(pilotsToSubmit, int(50 / self.maxJobsInFillMode)), 1) pilotOptions = [] privateIfGenericTQ = self.privatePilotFraction > random.random() privateTQ = "PilotTypes" in taskQueueDict and "private" in [t.lower() for t in taskQueueDict["PilotTypes"]] forceGeneric = "ForceGeneric" in taskQueueDict submitPrivatePilot = (privateIfGenericTQ or privateTQ) and not forceGeneric if submitPrivatePilot: self.log.verbose("Submitting private pilots for TaskQueue %s" % taskQueueDict["TaskQueueID"]) ownerDN = taskQueueDict["OwnerDN"] ownerGroup = taskQueueDict["OwnerGroup"] # User Group requirement pilotOptions.append("-G %s" % taskQueueDict["OwnerGroup"]) # check if group allows jobsharing ownerGroupProperties = getPropertiesForGroup(ownerGroup) if not "JobSharing" in ownerGroupProperties: # Add Owner requirement to pilot pilotOptions.append("-O '%s'" % ownerDN) if privateTQ: pilotOptions.append("-o /Resources/Computing/CEDefaults/PilotType=private") maxJobsInFillMode = self.maxJobsInFillMode else: # For generic jobs we'll submit mixture of generic and private pilots self.log.verbose("Submitting generic pilots for TaskQueue %s" % taskQueueDict["TaskQueueID"]) # ADRI: Find the generic group result = findGenericPilotCredentials(group=taskQueueDict["OwnerGroup"]) if not result["OK"]: self.log.error(ERROR_GENERIC_CREDENTIALS, result["Message"]) return S_ERROR(ERROR_GENERIC_CREDENTIALS) ownerDN, ownerGroup = result["Value"] result = gProxyManager.requestToken(ownerDN, ownerGroup, max(pilotsToSubmit, self.maxJobsInFillMode)) if not result["OK"]: self.log.error(ERROR_TOKEN, result["Message"]) return S_ERROR(ERROR_TOKEN) (token, numberOfUses) = result["Value"] pilotsToSubmit = min(numberOfUses, pilotsToSubmit) pilotOptions.append("-o /Security/ProxyToken=%s" % token) pilotsToSubmit = max(1, (pilotsToSubmit - 1) / self.maxJobsInFillMode + 1) maxJobsInFillMode = int(numberOfUses / pilotsToSubmit) # Use Filling mode pilotOptions.append("-M %s" % maxJobsInFillMode) # Debug pilotOptions.append("-d") # Setup. pilotOptions.append("-S %s" % taskQueueDict["Setup"]) # CS Servers csServers = gConfig.getServersList() if len(csServers) > 3: # Remove the master master = gConfigurationData.getMasterServer() if master in csServers: csServers.remove(master) pilotOptions.append("-C %s" % ",".join(csServers)) # DIRAC Extensions extensionsList = getCSExtensions() if extensionsList: pilotOptions.append("-e %s" % ",".join(extensionsList)) # Get DIRAC version and project, There might be global Setup defaults and per VO/Setup defaults (from configure) opsHelper = Operations(group=taskQueueDict["OwnerGroup"], setup=taskQueueDict["Setup"]) # Requested version of DIRAC (it can be a list, so we take the fist one) version = opsHelper.getValue(cfgPath("Pilot", "Version"), [self.installVersion])[0] pilotOptions.append("-r %s" % version) # Requested Project to install installProject = opsHelper.getValue(cfgPath("Pilot", "Project"), self.installProject) if installProject: pilotOptions.append("-l %s" % installProject) installation = opsHelper.getValue(cfgPath("Pilot", "Installation"), self.installation) if installation: pilotOptions.append("-V %s" % installation) # Requested CPU time pilotOptions.append("-T %s" % taskQueueDict["CPUTime"]) if self.submitPoolOption not in self.extraPilotOptions: pilotOptions.append(self.submitPoolOption) if self.extraPilotOptions: pilotOptions.extend(self.extraPilotOptions) return S_OK((pilotOptions, pilotsToSubmit, ownerDN, ownerGroup, submitPrivatePilot, privateTQ))
"" ])) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() result = getProxyInfo() if not result["OK"]: gLogger.error("Failed to get proxy information", result["Message"]) DIRAC.exit(2) proxy = result["Value"] if proxy["secondsLeft"] < 1: gLogger.error("Your proxy has expired, please create new one") DIRAC.exit(2) group = proxy["group"] if not "CSAdministrator" in getPropertiesForGroup(group): gLogger.error("You must be CSAdministrator user to execute this script") gLogger.notice( "Please issue 'dirac-proxy-init -g [group with CSAdministrator Property]'" ) DIRAC.exit(2) cs = CSAPI() result = cs.getCurrentCFG() if not result["OK"]: gLogger.error("Failed to get copy of CS", result["Message"]) DIRAC.exit(2) cfg = result["Value"] if not cfg.isSection("Resources"): gLogger.error("Section '/Resources' is absent in CS")
def _getPilotOptions( self, taskQueueDict, pilotsToSubmit ): # Need to limit the maximum number of pilots to submit at once # For generic pilots this is limited by the number of use of the tokens and the # maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation: pilotsToSubmit = min( pilotsToSubmit, int( 50 / self.maxJobsInFillMode ) ) pilotOptions = [ "-V %s" % self.virtualOrganization ] privateIfGenericTQ = self.privatePilotFraction > random.random() privateTQ = ( 'PilotTypes' in taskQueueDict and 'private' in [ t.lower() for t in taskQueueDict['PilotTypes'] ] ) forceGeneric = 'ForceGeneric' in taskQueueDict submitPrivatePilot = ( privateIfGenericTQ or privateTQ ) and not forceGeneric if submitPrivatePilot: self.log.verbose( 'Submitting private pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] ) ownerDN = taskQueueDict['OwnerDN'] ownerGroup = taskQueueDict['OwnerGroup'] # User Group requirement pilotOptions.append( '-G %s' % taskQueueDict['OwnerGroup'] ) # check if group allows jobsharing ownerGroupProperties = getPropertiesForGroup( ownerGroup ) if not 'JobSharing' in ownerGroupProperties: # Add Owner requirement to pilot pilotOptions.append( "-O '%s'" % ownerDN ) if privateTQ: pilotOptions.append( '-o /Resources/Computing/CEDefaults/PilotType=private' ) maxJobsInFillMode = self.maxJobsInFillMode else: #For generic jobs we'll submit mixture of generic and private pilots self.log.verbose( 'Submitting generic pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] ) ownerDN = self.genericPilotDN ownerGroup = self.genericPilotGroup result = gProxyManager.requestToken( ownerDN, ownerGroup, max( pilotsToSubmit, self.maxJobsInFillMode ) ) if not result[ 'OK' ]: self.log.error( ERROR_TOKEN, result['Message'] ) return S_ERROR( ERROR_TOKEN ) ( token, numberOfUses ) = result[ 'Value' ] pilotsToSubmit = min( numberOfUses, pilotsToSubmit ) pilotOptions.append( '-o /Security/ProxyToken=%s' % token ) pilotsToSubmit = ( pilotsToSubmit - 1 ) / self.maxJobsInFillMode + 1 maxJobsInFillMode = int( numberOfUses / pilotsToSubmit ) # Use Filling mode pilotOptions.append( '-M %s' % maxJobsInFillMode ) # Debug pilotOptions.append( '-d' ) # Setup. pilotOptions.append( '-S %s' % taskQueueDict['Setup'] ) # CS Servers csServers = gConfig.getValue( "/DIRAC/Configuration/Servers", [] ) pilotOptions.append( '-C %s' % ",".join( csServers ) ) # DIRAC Extensions extensionsList = getCSExtensions() if extensionsList: pilotOptions.append( '-e %s' % ",".join( extensionsList ) ) # Requested version of DIRAC pilotOptions.append( '-r %s' % self.installVersion ) # Requested Project to install pilotOptions.append( '-V %s' % self.installInstallation ) # Requested CPU time pilotOptions.append( '-T %s' % taskQueueDict['CPUTime'] ) if self.extraPilotOptions: pilotOptions.extend( self.extraPilotOptions ) return S_OK( ( pilotOptions, pilotsToSubmit, ownerDN, ownerGroup, submitPrivatePilot, privateTQ ) )
def _getPilotOptions( self, taskQueueDict, pilotsToSubmit ): # Need to limit the maximum number of pilots to submit at once # For generic pilots this is limited by the number of use of the tokens and the # maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation: pilotsToSubmit = max( min( pilotsToSubmit, int( 50 / self.maxJobsInFillMode ) ), 1 ) pilotOptions = [] privateIfGenericTQ = self.privatePilotFraction > random.random() privateTQ = ( 'PilotTypes' in taskQueueDict and 'private' in [ t.lower() for t in taskQueueDict['PilotTypes'] ] ) forceGeneric = 'ForceGeneric' in taskQueueDict submitPrivatePilot = ( privateIfGenericTQ or privateTQ ) and not forceGeneric if submitPrivatePilot: self.log.verbose( 'Submitting private pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] ) ownerDN = taskQueueDict['OwnerDN'] ownerGroup = taskQueueDict['OwnerGroup'] # User Group requirement pilotOptions.append( '-G %s' % taskQueueDict['OwnerGroup'] ) # check if group allows jobsharing ownerGroupProperties = getPropertiesForGroup( ownerGroup ) if not 'JobSharing' in ownerGroupProperties: # Add Owner requirement to pilot pilotOptions.append( "-O '%s'" % ownerDN ) if privateTQ: pilotOptions.append( '-o /Resources/Computing/CEDefaults/PilotType=private' ) maxJobsInFillMode = self.maxJobsInFillMode else: #For generic jobs we'll submit mixture of generic and private pilots self.log.verbose( 'Submitting generic pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] ) #ADRI: Find the generic group result = findGenericPilotCredentials( group = taskQueueDict[ 'OwnerGroup' ] ) if not result[ 'OK' ]: self.log.error( ERROR_GENERIC_CREDENTIALS, result[ 'Message' ] ) return S_ERROR( ERROR_GENERIC_CREDENTIALS ) ownerDN, ownerGroup = result[ 'Value' ] result = gProxyManager.requestToken( ownerDN, ownerGroup, max( pilotsToSubmit, self.maxJobsInFillMode ) ) if not result[ 'OK' ]: self.log.error( ERROR_TOKEN, result['Message'] ) return S_ERROR( ERROR_TOKEN ) ( token, numberOfUses ) = result[ 'Value' ] pilotsToSubmit = min( numberOfUses, pilotsToSubmit ) pilotOptions.append( '-o /Security/ProxyToken=%s' % token ) pilotsToSubmit = max( 1, ( pilotsToSubmit - 1 ) / self.maxJobsInFillMode + 1 ) maxJobsInFillMode = int( numberOfUses / pilotsToSubmit ) # Use Filling mode pilotOptions.append( '-M %s' % maxJobsInFillMode ) # Debug pilotOptions.append( '-d' ) # Setup. pilotOptions.append( '-S %s' % taskQueueDict['Setup'] ) # CS Servers csServers = gConfig.getServersList() if len( csServers ) > 3: # Remove the master master = gConfigurationData.getMasterServer() if master in csServers: csServers.remove( master ) pilotOptions.append( '-C %s' % ",".join( csServers ) ) # DIRAC Extensions to be used in pilots # ubeda: I'm not entirely sure if we can use here the same opsHelper as in line # line +352 pilotExtensionsList = Operations().getValue( "Pilot/Extensions", [] ) extensionsList = [] if pilotExtensionsList: if pilotExtensionsList[0] != 'None': extensionsList = pilotExtensionsList else: extensionsList = getCSExtensions() if extensionsList: pilotOptions.append( '-e %s' % ",".join( extensionsList ) ) #Get DIRAC version and project, There might be global Setup defaults and per VO/Setup defaults (from configure) opsHelper = Operations( group = taskQueueDict['OwnerGroup'], setup = taskQueueDict['Setup'] ) # Requested version of DIRAC (it can be a list, so we take the fist one) version = opsHelper.getValue( cfgPath( 'Pilot', 'Version' ) , [ self.installVersion ] )[0] pilotOptions.append( '-r %s' % version ) # Requested Project to install installProject = opsHelper.getValue( cfgPath( 'Pilot', 'Project' ) , self.installProject ) if installProject: pilotOptions.append( '-l %s' % installProject ) installation = opsHelper.getValue( cfgPath( 'Pilot', 'Installation' ), self.installation ) if installation: pilotOptions.append( "-V %s" % installation ) # Requested CPU time pilotOptions.append( '-T %s' % taskQueueDict['CPUTime'] ) if self.submitPoolOption not in self.extraPilotOptions: pilotOptions.append( self.submitPoolOption ) if self.extraPilotOptions: pilotOptions.extend( self.extraPilotOptions ) return S_OK( ( pilotOptions, pilotsToSubmit, ownerDN, ownerGroup, submitPrivatePilot, privateTQ ) )
def _getPilotOptions(self, taskQueueDict, pilotsToSubmit): # Need to limit the maximum number of pilots to submit at once # For generic pilots this is limited by the number of use of the tokens and the # maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation: pilotsToSubmit = max( min(pilotsToSubmit, int(50 / self.maxJobsInFillMode)), 1) pilotOptions = [] privateIfGenericTQ = self.privatePilotFraction > random.random() privateTQ = ('PilotTypes' in taskQueueDict and 'private' in [t.lower() for t in taskQueueDict['PilotTypes']]) forceGeneric = 'ForceGeneric' in taskQueueDict submitPrivatePilot = (privateIfGenericTQ or privateTQ) and not forceGeneric if submitPrivatePilot: self.log.verbose('Submitting private pilots for TaskQueue %s' % taskQueueDict['TaskQueueID']) ownerDN = taskQueueDict['OwnerDN'] ownerGroup = taskQueueDict['OwnerGroup'] # User Group requirement pilotOptions.append('-G %s' % taskQueueDict['OwnerGroup']) # check if group allows jobsharing ownerGroupProperties = getPropertiesForGroup(ownerGroup) if not 'JobSharing' in ownerGroupProperties: # Add Owner requirement to pilot pilotOptions.append("-O '%s'" % ownerDN) if privateTQ: pilotOptions.append( '-o /Resources/Computing/CEDefaults/PilotType=private') maxJobsInFillMode = self.maxJobsInFillMode else: #For generic jobs we'll submit mixture of generic and private pilots self.log.verbose('Submitting generic pilots for TaskQueue %s' % taskQueueDict['TaskQueueID']) #ADRI: Find the generic group result = findGenericPilotCredentials( group=taskQueueDict['OwnerGroup']) if not result['OK']: self.log.error(ERROR_GENERIC_CREDENTIALS, result['Message']) return S_ERROR(ERROR_GENERIC_CREDENTIALS) ownerDN, ownerGroup = result['Value'] result = gProxyManager.requestToken( ownerDN, ownerGroup, max(pilotsToSubmit, self.maxJobsInFillMode)) if not result['OK']: self.log.error(ERROR_TOKEN, result['Message']) return S_ERROR(ERROR_TOKEN) (token, numberOfUses) = result['Value'] pilotsToSubmit = min(numberOfUses, pilotsToSubmit) pilotOptions.append('-o /Security/ProxyToken=%s' % token) pilotsToSubmit = max( 1, (pilotsToSubmit - 1) / self.maxJobsInFillMode + 1) maxJobsInFillMode = int(numberOfUses / pilotsToSubmit) # Use Filling mode pilotOptions.append('-M %s' % maxJobsInFillMode) # Debug pilotOptions.append('-d') # Setup. pilotOptions.append('-S %s' % taskQueueDict['Setup']) # CS Servers csServers = gConfig.getServersList() if len(csServers) > 3: # Remove the master master = gConfigurationData.getMasterServer() if master in csServers: csServers.remove(master) pilotOptions.append('-C %s' % ",".join(csServers)) # DIRAC Extensions to be used in pilots # ubeda: I'm not entirely sure if we can use here the same opsHelper as in line # line +352 pilotExtensionsList = Operations().getValue("Pilot/Extensions", []) extensionsList = [] if pilotExtensionsList: if pilotExtensionsList[0] != 'None': extensionsList = pilotExtensionsList else: extensionsList = getCSExtensions() if extensionsList: pilotOptions.append('-e %s' % ",".join(extensionsList)) #Get DIRAC version and project, There might be global Setup defaults and per VO/Setup defaults (from configure) opsHelper = Operations(group=taskQueueDict['OwnerGroup'], setup=taskQueueDict['Setup']) # Requested version of DIRAC (it can be a list, so we take the fist one) version = opsHelper.getValue(cfgPath('Pilot', 'Version'), [self.installVersion])[0] pilotOptions.append('-r %s' % version) # Requested Project to install installProject = opsHelper.getValue(cfgPath('Pilot', 'Project'), self.installProject) if installProject: pilotOptions.append('-l %s' % installProject) installation = opsHelper.getValue(cfgPath('Pilot', 'Installation'), self.installation) if installation: pilotOptions.append("-V %s" % installation) # Requested CPU time pilotOptions.append('-T %s' % taskQueueDict['CPUTime']) if self.submitPoolOption not in self.extraPilotOptions: pilotOptions.append(self.submitPoolOption) if self.extraPilotOptions: pilotOptions.extend(self.extraPilotOptions) return S_OK((pilotOptions, pilotsToSubmit, ownerDN, ownerGroup, submitPrivatePilot, privateTQ))
"" ])) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() result = getProxyInfo() if not result["OK"]: gLogger.error("Failed to get proxy information", result["Message"]) DIRACExit(2) proxy = result["Value"] if proxy["secondsLeft"] < 1: gLogger.error("Your proxy has expired, please create new one") DIRACExit(2) group = proxy["group"] if "CSAdministrator" not in getPropertiesForGroup(group): gLogger.error("You must be CSAdministrator user to execute this script") gLogger.notice( "Please issue 'dirac-proxy-init -g [group with CSAdministrator Property]'" ) DIRACExit(2) cs = CSAPI() result = cs.getCurrentCFG() if not result["OK"]: gLogger.error("Failed to get copy of CS", result["Message"]) DIRACExit(2) cfg = result["Value"] if not cfg.isSection("Resources"): gLogger.error("Section '/Resources' is absent in CS")