def getGroupsToUpload( self ): uploadGroups = [] if self.__piParams.uploadProxy or Registry.getGroupOption( self.__piParams.diracGroup, "AutoUploadProxy", False ): uploadGroups.append( self.__piParams.diracGroup ) if not self.__piParams.uploadPilot: if not Registry.getGroupOption( self.__piParams.diracGroup, "AutoUploadPilotProxy", False ): return uploadGroups issuerCert = self.getIssuerCert() resultUserDN = issuerCert.getSubjectDN() #pylint: disable=no-member if not resultUserDN['OK']: return resultUserDN userDN = resultUserDN[ 'Value' ] resultGroups = Registry.getGroupsForDN( userDN ) if not resultGroups[ 'OK' ]: gLogger.error( "No groups defined for DN %s" % userDN ) return [] availableGroups = resultGroups[ 'Value' ] for group in availableGroups: groupProps = Registry.getPropertiesForGroup( group ) if Properties.PILOT in groupProps or Properties.GENERIC_PILOT in groupProps: uploadGroups.append( group ) return uploadGroups
def getGroupsToUpload(self): uploadGroups = [] if self.__piParams.uploadProxy or Registry.getGroupOption( self.__piParams.diracGroup, "AutoUploadProxy", False): uploadGroups.append(self.__piParams.diracGroup) if not self.__piParams.uploadPilot: if not Registry.getGroupOption(self.__piParams.diracGroup, "AutoUploadPilotProxy", False): return uploadGroups issuerCert = self.getIssuerCert() userDN = issuerCert.getSubjectDN()['Value'] result = Registry.getGroupsForDN(userDN) if not result['OK']: gLogger.error("No groups defined for DN %s" % userDN) return [] availableGroups = result['Value'] pilotGroups = [] for group in availableGroups: groupProps = Registry.getPropertiesForGroup(group) if Properties.PILOT in groupProps or Properties.GENERIC_PILOT in groupProps: uploadGroups.append(group) return uploadGroups
def addVOMSExtIfNeeded(self): addVOMS = self.__piParams.addVOMSExt or Registry.getGroupOption( self.__piParams.diracGroup, "AutoAddVOMS", False ) if not addVOMS: return S_OK() vomsAttr = Registry.getVOMSAttributeForGroup(self.__piParams.diracGroup) if not vomsAttr: return S_ERROR( "Requested adding a VOMS extension but no VOMS attribute defined for group %s" % self.__piParams.diracGroup ) result = VOMS.VOMS().setVOMSAttributes( self.__proxyGenerated, attribute=vomsAttr, vo=Registry.getVOForGroup(self.__piParams.diracGroup) ) if not result["OK"]: return S_ERROR( "Could not add VOMS extensions to the proxy\nFailed adding VOMS attribute: %s" % result["Message"] ) gLogger.notice("Added VOMS attribute %s" % vomsAttr) chain = result["Value"] chain.dumpAllToFile(self.__proxyGenerated) return S_OK()
def addVOMSExtIfNeeded(self): addVOMS = self.__piParams.addVOMSExt or Registry.getGroupOption( self.__piParams.diracGroup, "AutoAddVOMS", False) if not addVOMS: return S_OK() vomsAttr = Registry.getVOMSAttributeForGroup( self.__piParams.diracGroup) if not vomsAttr: return S_ERROR( "Requested adding a VOMS extension but no VOMS attribute defined for group %s" % self.__piParams.diracGroup) result = VOMS.VOMS().setVOMSAttributes(self.__proxyGenerated, attribute=vomsAttr, vo=Registry.getVOMSVOForGroup( self.__piParams.diracGroup)) if not result['OK']: return S_ERROR( "Could not add VOMS extensions to the proxy\nFailed adding VOMS attribute: %s" % result['Message']) gLogger.notice("Added VOMS attribute %s" % vomsAttr) chain = result['Value'] chain.dumpAllToFile(self.__proxyGenerated) return S_OK()
def setMetadata(self, dPath, metaDict, credDict): """Set the value of a given metadata field for the the given directory path :param str dPath: directory path :param dict metaDict: dictionary with metadata :param dict credDict: client credential dictionary :return: S_OK/S_ERROR """ result = self._getMetadataFields(credDict) if not result["OK"]: return result metaFields = result["Value"] result = self.db.dtree.findDir(dPath) if not result["OK"]: return result if not result["Value"]: return S_ERROR("Path not found: %s" % dPath) dirID = result["Value"] dirmeta = self.getDirectoryMetadata(dPath, credDict, ownData=False) if not dirmeta["OK"]: return dirmeta voName = Registry.getGroupOption(credDict["group"], "VO") forceIndex = Operations(vo=voName).getValue( "DataManagement/ForceIndexedMetadata", False) for metaName, metaValue in metaDict.items(): if metaName not in metaFields: if forceIndex: return S_ERROR( "Field %s not indexed, but ForceIndexedMetadata is set" % metaName, callStack=[]) result = self.setMetaParameter(dPath, metaName, metaValue, credDict) if not result["OK"]: return result continue # Check that the metadata is not defined for the parent directories if metaName in dirmeta["Value"]: return S_ERROR( "Metadata conflict detected for %s for directory %s" % (metaName, dPath)) result = self.db.insertFields("FC_Meta_%s" % metaName, ["DirID", "Value"], [dirID, metaValue]) if not result["OK"]: if result["Message"].find("Duplicate") != -1: req = "UPDATE FC_Meta_%s SET Value='%s' WHERE DirID=%d" % ( metaName, metaValue, dirID) result = self.db._update(req) if not result["OK"]: return result else: return result return S_OK()
def sendExpirationNotifications( self ): result = self.__cleanExpNotifs() if not result[ 'OK' ]: return result cmd = "SELECT UserDN, UserGroup, LifeLimit FROM `ProxyDB_ExpNotifs`" result = self._query( cmd ) if not result[ 'OK' ]: return result notifDone = dict( [ ( ( row[0], row[1] ), row[2] ) for row in result[ 'Value' ] ] ) notifLimits = sorted( [ int( x ) for x in self.getCSOption( "NotificationTimes", ProxyDB.NOTIFICATION_TIMES ) ] ) sqlSel = "UserDN, UserGroup, TIMESTAMPDIFF( SECOND, UTC_TIMESTAMP(), ExpirationTime )" sqlCond = "TIMESTAMPDIFF( SECOND, UTC_TIMESTAMP(), ExpirationTime ) < %d" % max( notifLimits ) cmd = "SELECT %s FROM `ProxyDB_Proxies` WHERE %s" % ( sqlSel, sqlCond ) result = self._query( cmd ) if not result[ 'OK' ]: return result pilotProps = ( Properties.GENERIC_PILOT, Properties.PILOT ) data = result[ 'Value' ] sent = [] for row in data: userDN, group, lTime = row #If it's a pilot proxy, skip it if Registry.groupHasProperties( group, pilotProps ): continue #IF it dosn't hace the auto upload proxy, skip it if not Registry.getGroupOption( group, "AutoUploadProxy", False ): continue notKey = ( userDN, group ) for notifLimit in notifLimits: if notifLimit < lTime: #Not yet in this notification limit continue if notKey in notifDone and notifDone[ notKey ] <= notifLimit: #Already notified for this notification limit break if not self._notifyProxyAboutToExpire( userDN, group, lTime, notifLimit ): #Cannot send notification, retry later break try: sUserDN = self._escapeString( userDN )[ 'Value' ] sGroup = self._escapeString( group )[ 'Value' ] except KeyError: return S_ERROR( "OOPS" ) if notKey not in notifDone: values = "( %s, %s, %d, TIMESTAMPADD( SECOND, %s, UTC_TIMESTAMP() ) )" % ( sUserDN, sGroup, notifLimit, lTime ) cmd = "INSERT INTO `ProxyDB_ExpNotifs` ( UserDN, UserGroup, LifeLimit, ExpirationTime ) VALUES %s" % values result = self._update( cmd ) if not result[ 'OK' ]: gLogger.error( "Could not mark notification as sent", result[ 'Message' ] ) else: values = "LifeLimit = %d, ExpirationTime = TIMESTAMPADD( SECOND, %s, UTC_TIMESTAMP() )" % ( notifLimit, lTime ) cmd = "UPDATE `ProxyDB_ExpNotifs` SET %s WHERE UserDN = %s AND UserGroup = %s" % ( values, sUserDN, sGroup ) result = self._update( cmd ) if not result[ 'OK' ]: gLogger.error( "Could not mark notification as sent", result[ 'Message' ] ) sent.append( ( userDN, group, lTime ) ) notifDone[ notKey ] = notifLimit return S_OK( sent )
def _getMetaNameSuffix(credDict): """ Get a VO specific suffix from user credentials. :param credDict: user credentials :return: VO specific suffix """ vo = Registry.getGroupOption(credDict['group'], 'VO') return VO_SUFFIX_SEPARATOR + vo.replace('-', '_').replace('.', '_')
def _getMetaNameSuffix(credDict): """ Get a VO specific suffix from user credentials. :param credDict: user credentials :return: VO specific suffix """ vo = Registry.getGroupOption(credDict["group"], "VO") return VO_SUFFIX_SEPARATOR + vo.replace("-", "_").replace(".", "_")
def certLifeTimeCheck(self): minLife = Registry.getGroupOption(self.__piParams.diracGroup, "SafeCertificateLifeTime", 2592000) resultIssuerCert = self.getIssuerCert() resultRemainingSecs = resultIssuerCert.getRemainingSecs() # pylint: disable=no-member if not resultRemainingSecs["OK"]: gLogger.error("Could not retrieve certificate expiration time", resultRemainingSecs["Message"]) return lifeLeft = resultRemainingSecs["Value"] if minLife > lifeLeft: daysLeft = int(lifeLeft / 86400) msg = "Your certificate will expire in less than %d days. Please renew it!" % daysLeft sep = "=" * (len(msg) + 4) msg = "%s\n %s \n%s" % (sep, msg, sep) gLogger.notice(msg)
def certLifeTimeCheck( self ): minLife = Registry.getGroupOption( self.__piParams.diracGroup, "SafeCertificateLifeTime", 2592000 ) issuerCert = self.getIssuerCert() result = issuerCert.getRemainingSecs() if not result[ 'OK' ]: gLogger.error( "Could not retrieve certificate expiration time", result[ 'Message' ] ) return lifeLeft = result[ 'Value' ] if minLife > lifeLeft: daysLeft = int( lifeLeft / 86400 ) msg = "Your certificate will expire in less than %d days. Please renew it!" % daysLeft sep = "=" * ( len( msg ) + 4 ) msg = "%s\n %s \n%s" % ( sep, msg, sep ) gLogger.notice( msg )
def setMetadata(self, path, metaDict, credDict): """Set the value of a given metadata field for the the given directory path :param str path: file path :param dict metaDict: dictionary with metadata :param dict credDict: client credential dictionary :return: S_OK/S_ERROR """ result = self._getFileMetadataFields(credDict) if not result["OK"]: return result metaFields = result["Value"] result = self.db.fileManager._findFiles([path]) if not result["OK"]: return result if result["Value"]["Successful"]: fileID = result["Value"]["Successful"][path]["FileID"] else: return S_ERROR("File %s not found" % path) voName = Registry.getGroupOption(credDict["group"], "VO") forceIndex = Operations(vo=voName).getValue( "DataManagement/ForceIndexedMetadata", False) for metaName, metaValue in metaDict.items(): if metaName not in metaFields: if forceIndex: return S_ERROR( "Field %s not indexed, but ForceIndexedMetadata is set" % metaName, callStack=[]) result = self.__setFileMetaParameter(fileID, metaName, metaValue, credDict) else: result = self.db.insertFields("FC_FileMeta_%s" % metaName, ["FileID", "Value"], [fileID, metaValue]) if not result["OK"]: if result["Message"].find("Duplicate") != -1: req = "UPDATE FC_FileMeta_%s SET Value='%s' WHERE FileID=%d" % ( metaName, metaValue, fileID) result = self.db._update(req) if not result["OK"]: return result else: return result return S_OK()
def addVOMSExtIfNeeded( self ): addVOMS = Registry.getGroupOption( self.__piParams.diracGroup, "AutoAddVOMS", self.__piParams.addVOMSExt ) if not addVOMS: return vomsAttr = Registry.getVOMSAttributeForGroup( self.__piParams.diracGroup ) if not vomsAttr: gLogger.error( "Requested adding a VOMS extension but no VOMS attribute defined for group %s" % self.__piParams.diracGroup ) return result = VOMS.VOMS().setVOMSAttributes( vomsAttr, vo = Registry.getVOForGroup( self.__piParams.diracGroup ) ) if not result[ 'OK' ]: gLogger.error( "Failed adding VOMS attribute:", result[ 'Message' ] ) return False else: gLogger.notice( "Added VOMS attribute %s" % vomsAttr )
def getPilotGroupsToUpload( self ): pilotUpload = Registry.getGroupOption( self.__piParams.diracGroup, "AutoUploadPilotProxy", self.__piParams.uploadPilot ) if not pilotUpload: return [] issuerCert = self.getIssuerCert() userDN = issuerCert.getSubjectDN()[ 'Value' ] result = Registry.getGroupsForDN( userDN ) if not result[ 'OK' ]: gLogger.error( "No groups defined for DN %s" % userDN ) return [] availableGroups = result[ 'Value' ] pilotGroups = [] for group in availableGroups: groupProps = Registry.getPropertiesForGroup( group ) if Properties.PILOT in groupProps or Properties.GENERIC_PILOT in groupProps: pilotGroups.append( group ) return pilotGroups
def __checkParams( self ): #Check if the proxy needs to be uploaded if not self.__piParams.uploadProxy: self.__piParams.uploadProxy = Registry.getGroupOption( self.__piParams.diracGroup, "AutoUploadProxy", False ) if self.__piParams.uploadProxy: gLogger.verbose( "Proxy will be uploaded to ProxyManager" )
def beginExecution(self): self.gridEnv = self.am_getOption("GridEnv", getGridEnv()) # The SiteDirector is for a particular user community self.vo = self.am_getOption("Community", '') if not self.vo: self.vo = CSGlobals.getVO() # The SiteDirector is for a particular user group self.group = self.am_getOption("Group", '') # self.voGroups contain all the eligible user groups for pilots submutted by this SiteDirector self.voGroups = [] # Choose the group for which pilots will be submitted. This is a hack until # we will be able to match pilots to VOs. if not self.group: if self.vo: result = Registry.getGroupsForVO(self.vo) if not result['OK']: return result for group in result['Value']: if 'NormalUser' in Registry.getPropertiesForGroup(group): self.voGroups.append(group) else: self.voGroups = [self.group] result = findGenericPilotCredentials(vo=self.vo) if not result['OK']: return result self.pilotDN, self.pilotGroup = result['Value'] self.pilotDN = self.am_getOption("PilotDN", self.pilotDN) self.pilotGroup = self.am_getOption("PilotGroup", self.pilotGroup) self.platforms = [] self.sites = [] self.defaultSubmitPools = '' if self.group: self.defaultSubmitPools = Registry.getGroupOption( self.group, 'SubmitPools', '') elif self.vo: self.defaultSubmitPools = Registry.getVOOption( self.vo, 'SubmitPools', '') self.pilot = self.am_getOption('PilotScript', DIRAC_PILOT) self.install = DIRAC_INSTALL self.workingDirectory = self.am_getOption('WorkDirectory') self.maxQueueLength = self.am_getOption('MaxQueueLength', 86400 * 3) self.pilotLogLevel = self.am_getOption('PilotLogLevel', 'INFO') self.maxJobsInFillMode = self.am_getOption('MaxJobsInFillMode', self.maxJobsInFillMode) self.maxPilotsToSubmit = self.am_getOption('MaxPilotsToSubmit', self.maxPilotsToSubmit) self.pilotWaitingFlag = self.am_getOption('PilotWaitingFlag', True) self.pilotWaitingTime = self.am_getOption('MaxPilotWaitingTime', 7200) # Flags self.updateStatus = self.am_getOption('UpdatePilotStatus', True) self.getOutput = self.am_getOption('GetPilotOutput', True) self.sendAccounting = self.am_getOption('SendPilotAccounting', True) # Get the site description dictionary siteNames = None if not self.am_getOption('Site', 'Any').lower() == "any": siteNames = self.am_getOption('Site', []) ceTypes = None if not self.am_getOption('CETypes', 'Any').lower() == "any": ceTypes = self.am_getOption('CETypes', []) ces = None if not self.am_getOption('CEs', 'Any').lower() == "any": ces = self.am_getOption('CEs', []) result = Resources.getQueues(community=self.vo, siteList=siteNames, ceList=ces, ceTypeList=ceTypes, mode='Direct') if not result['OK']: return result resourceDict = result['Value'] result = self.getQueues(resourceDict) if not result['OK']: return result #if not siteNames: # siteName = gConfig.getValue( '/DIRAC/Site', 'Unknown' ) # if siteName == 'Unknown': # return S_OK( 'No site specified for the SiteDirector' ) # else: # siteNames = [siteName] #self.siteNames = siteNames if self.updateStatus: self.log.always('Pilot status update requested') if self.getOutput: self.log.always('Pilot output retrieval requested') if self.sendAccounting: self.log.always('Pilot accounting sending requested') self.log.always('Sites:', siteNames) self.log.always('CETypes:', ceTypes) self.log.always('CEs:', ces) self.log.always('PilotDN:', self.pilotDN) self.log.always('PilotGroup:', self.pilotGroup) self.log.always('MaxPilotsToSubmit:', self.maxPilotsToSubmit) self.log.always('MaxJobsInFillMode:', self.maxJobsInFillMode) self.localhost = socket.getfqdn() self.proxy = '' if self.queueDict: self.log.always("Agent will serve queues:") for queue in self.queueDict: self.log.always("Site: %s, CE: %s, Queue: %s" % (self.queueDict[queue]['Site'], self.queueDict[queue]['CEName'], queue)) return S_OK()
def generateProxy( params ): if params.checkClock: result = getClockDeviation() if result[ 'OK' ]: deviation = result[ 'Value' ] if deviation > 600: gLogger.error( "Your host clock seems to be off by more than TEN MINUTES! Thats really bad." ) gLogger.error( "We're cowardly refusing to generate a proxy. Please fix your system time" ) sys.exit( 1 ) elif deviation > 180: gLogger.error( "Your host clock seems to be off by more than THREE minutes! Thats bad." ) gLogger.notice( "We'll generate the proxy but please fix your system time" ) elif deviation > 60: gLogger.error( "Your host clock seems to be off by more than a minute! Thats not good." ) gLogger.notice( "We'll generate the proxy but please fix your system time" ) certLoc = params.certLoc keyLoc = params.keyLoc if not certLoc or not keyLoc: cakLoc = Locations.getCertificateAndKeyLocation() if not cakLoc: return S_ERROR( "Can't find user certificate and key" ) if not certLoc: certLoc = cakLoc[0] if not keyLoc: keyLoc = cakLoc[1] params.certLoc = certLoc params.keyLoc = keyLoc #Load password testChain = X509Chain() retVal = testChain.loadKeyFromFile( keyLoc, password = params.userPasswd ) if not retVal[ 'OK' ]: passwdPrompt = "Enter Certificate password:"******"\n" ) else: userPasswd = getpass.getpass( passwdPrompt ) params.userPasswd = userPasswd #Find location proxyLoc = params.proxyLoc if not proxyLoc: proxyLoc = Locations.getDefaultProxyLocation() chain = X509Chain() #Load user cert and key retVal = chain.loadChainFromFile( certLoc ) if not retVal[ 'OK' ]: gLogger.warn( retVal[ 'Message' ] ) return S_ERROR( "Can't load %s" % certLoc ) retVal = chain.loadKeyFromFile( keyLoc, password = params.userPasswd ) if not retVal[ 'OK' ]: gLogger.warn( retVal[ 'Message' ] ) return S_ERROR( "Can't load %s" % keyLoc ) if params.checkWithCS: retVal = chain.generateProxyToFile( proxyLoc, params.proxyLifeTime, strength = params.proxyStrength, limited = params.limitedProxy ) gLogger.info( "Contacting CS..." ) retVal = Script.enableCS() if not retVal[ 'OK' ]: gLogger.warn( retVal[ 'Message' ] ) return S_ERROR( "Can't contact DIRAC CS: %s" % retVal[ 'Message' ] ) userDN = chain.getCertInChain( -1 )['Value'].getSubjectDN()['Value'] if not params.diracGroup: result = Registry.findDefaultGroupForDN( userDN ) if not result[ 'OK' ]: gLogger.warn( "Could not get a default group for DN %s: %s" % ( userDN, result[ 'Message' ] ) ) else: params.diracGroup = result[ 'Value' ] gLogger.info( "Default discovered group is %s" % params.diracGroup ) gLogger.info( "Checking DN %s" % userDN ) retVal = Registry.getUsernameForDN( userDN ) if not retVal[ 'OK' ]: gLogger.warn( retVal[ 'Message' ] ) return S_ERROR( "DN %s is not registered" % userDN ) username = retVal[ 'Value' ] gLogger.info( "Username is %s" % username ) retVal = Registry.getGroupsForUser( username ) if not retVal[ 'OK' ]: gLogger.warn( retVal[ 'Message' ] ) return S_ERROR( "User %s has no groups defined" % username ) groups = retVal[ 'Value' ] if params.diracGroup not in groups: return S_ERROR( "Requested group %s is not valid for user %s" % ( params.diracGroup, username ) ) gLogger.info( "Creating proxy for %s@%s (%s)" % ( username, params.diracGroup, userDN ) ) #Check if the proxy needs to be uploaded if not params.uploadProxy: params.uploadProxy = Registry.getGroupOption( params.diracGroup, "AutoUploadProxy", False ) gLogger.verbose( "Proxy will be uploaded to ProxyManager " ) if params.summary: h = int( params.proxyLifeTime / 3600 ) m = int( params.proxyLifeTime / 60 ) - h * 60 gLogger.notice( "Proxy lifetime will be %02d:%02d" % ( h, m ) ) gLogger.notice( "User cert is %s" % certLoc ) gLogger.notice( "User key is %s" % keyLoc ) gLogger.notice( "Proxy will be written to %s" % proxyLoc ) if params.diracGroup: gLogger.notice( "DIRAC Group will be set to %s" % params.diracGroup ) else: gLogger.notice( "No DIRAC Group will be set" ) gLogger.notice( "Proxy strength will be %s" % params.proxyStrength ) if params.limitedProxy: gLogger.notice( "Proxy will be limited" ) retVal = chain.generateProxyToFile( proxyLoc, params.proxyLifeTime, params.diracGroup, strength = params.proxyStrength, limited = params.limitedProxy ) if not retVal[ 'OK' ]: gLogger.warn( retVal[ 'Message' ] ) return S_ERROR( "Couldn't generate proxy: %s" % retVal[ 'Message' ] ) return S_OK( proxyLoc )
def getSubmitPools( group = None, vo = None ): if group: return Registry.getGroupOption( group, 'SubmitPools', '' ) if vo: return Registry.getVOOption( vo, 'SubmitPools', '' ) return ''
def beginExecution(self): self.gridEnv = self.am_getOption("GridEnv", getGridEnv()) # The SiteDirector is for a particular user community self.vo = self.am_getOption("Community", "") if not self.vo: self.vo = CSGlobals.getVO() # The SiteDirector is for a particular user group self.group = self.am_getOption("Group", "") # self.voGroups contain all the eligible user groups for pilots submutted by this SiteDirector self.voGroups = [] # Choose the group for which pilots will be submitted. This is a hack until # we will be able to match pilots to VOs. if not self.group: if self.vo: result = Registry.getGroupsForVO(self.vo) if not result["OK"]: return result for group in result["Value"]: if "NormalUser" in Registry.getPropertiesForGroup(group): self.voGroups.append(group) else: self.voGroups = [self.group] result = findGenericPilotCredentials(vo=self.vo) if not result["OK"]: return result self.pilotDN, self.pilotGroup = result["Value"] self.pilotDN = self.am_getOption("PilotDN", self.pilotDN) self.pilotGroup = self.am_getOption("PilotGroup", self.pilotGroup) self.platforms = [] self.sites = [] self.defaultSubmitPools = "" if self.group: self.defaultSubmitPools = Registry.getGroupOption(self.group, "SubmitPools", "") elif self.vo: self.defaultSubmitPools = Registry.getVOOption(self.vo, "SubmitPools", "") self.pilot = self.am_getOption("PilotScript", DIRAC_PILOT) self.install = DIRAC_INSTALL self.workingDirectory = self.am_getOption("WorkDirectory") self.maxQueueLength = self.am_getOption("MaxQueueLength", 86400 * 3) self.pilotLogLevel = self.am_getOption("PilotLogLevel", "INFO") self.maxJobsInFillMode = self.am_getOption("MaxJobsInFillMode", self.maxJobsInFillMode) self.maxPilotsToSubmit = self.am_getOption("MaxPilotsToSubmit", self.maxPilotsToSubmit) self.pilotWaitingFlag = self.am_getOption("PilotWaitingFlag", True) self.pilotWaitingTime = self.am_getOption("MaxPilotWaitingTime", 7200) # Flags self.updateStatus = self.am_getOption("UpdatePilotStatus", True) self.getOutput = self.am_getOption("GetPilotOutput", True) self.sendAccounting = self.am_getOption("SendPilotAccounting", True) # Get the site description dictionary siteNames = None if not self.am_getOption("Site", "Any").lower() == "any": siteNames = self.am_getOption("Site", []) ceTypes = None if not self.am_getOption("CETypes", "Any").lower() == "any": ceTypes = self.am_getOption("CETypes", []) ces = None if not self.am_getOption("CEs", "Any").lower() == "any": ces = self.am_getOption("CEs", []) result = Resources.getQueues( community=self.vo, siteList=siteNames, ceList=ces, ceTypeList=ceTypes, mode="Direct" ) if not result["OK"]: return result resourceDict = result["Value"] result = self.getQueues(resourceDict) if not result["OK"]: return result # if not siteNames: # siteName = gConfig.getValue( '/DIRAC/Site', 'Unknown' ) # if siteName == 'Unknown': # return S_OK( 'No site specified for the SiteDirector' ) # else: # siteNames = [siteName] # self.siteNames = siteNames if self.updateStatus: self.log.always("Pilot status update requested") if self.getOutput: self.log.always("Pilot output retrieval requested") if self.sendAccounting: self.log.always("Pilot accounting sending requested") self.log.always("Sites:", siteNames) self.log.always("CETypes:", ceTypes) self.log.always("CEs:", ces) self.log.always("PilotDN:", self.pilotDN) self.log.always("PilotGroup:", self.pilotGroup) self.log.always("MaxPilotsToSubmit:", self.maxPilotsToSubmit) self.log.always("MaxJobsInFillMode:", self.maxJobsInFillMode) self.localhost = socket.getfqdn() self.proxy = "" if self.queueDict: self.log.always("Agent will serve queues:") for queue in self.queueDict: self.log.always( "Site: %s, CE: %s, Queue: %s" % (self.queueDict[queue]["Site"], self.queueDict[queue]["CEName"], queue) ) return S_OK()
def getSubmitPools( group = None, vo = None ): if group: return Registry.getGroupOption( group, 'SubmitPools', '' ) if vo: return Registry.getVOOption( vo, 'SubmitPools', '' ) return ''
def beginExecution( self ): self.gridEnv = self.am_getOption( "GridEnv", getGridEnv() ) # The SiteDirector is for a particular user community self.vo = self.am_getOption( "VO", '' ) if not self.vo: self.vo = self.am_getOption( "Community", '' ) if not self.vo: self.vo = CSGlobals.getVO() # The SiteDirector is for a particular user group self.group = self.am_getOption( "Group", '' ) # self.voGroups contain all the eligible user groups for pilots submutted by this SiteDirector self.voGroups = [] # Choose the group for which pilots will be submitted. This is a hack until # we will be able to match pilots to VOs. if not self.group: if self.vo: result = Registry.getGroupsForVO( self.vo ) if not result['OK']: return result for group in result['Value']: if 'NormalUser' in Registry.getPropertiesForGroup( group ): self.voGroups.append( group ) else: self.voGroups = [ self.group ] result = findGenericPilotCredentials( vo = self.vo ) if not result[ 'OK' ]: return result self.pilotDN, self.pilotGroup = result[ 'Value' ] self.pilotDN = self.am_getOption( "PilotDN", self.pilotDN ) self.pilotGroup = self.am_getOption( "PilotGroup", self.pilotGroup ) self.platforms = [] self.sites = [] self.defaultSubmitPools = '' if self.group: self.defaultSubmitPools = Registry.getGroupOption( self.group, 'SubmitPools', '' ) elif self.vo: self.defaultSubmitPools = Registry.getVOOption( self.vo, 'SubmitPools', '' ) self.pilot = self.am_getOption( 'PilotScript', DIRAC_PILOT ) self.install = DIRAC_INSTALL self.extraModules = self.am_getOption( 'ExtraPilotModules', [] ) + DIRAC_MODULES self.workingDirectory = self.am_getOption( 'WorkDirectory' ) self.maxQueueLength = self.am_getOption( 'MaxQueueLength', 86400 * 3 ) self.pilotLogLevel = self.am_getOption( 'PilotLogLevel', 'INFO' ) self.maxJobsInFillMode = self.am_getOption( 'MaxJobsInFillMode', self.maxJobsInFillMode ) self.maxPilotsToSubmit = self.am_getOption( 'MaxPilotsToSubmit', self.maxPilotsToSubmit ) self.pilotWaitingFlag = self.am_getOption( 'PilotWaitingFlag', True ) self.pilotWaitingTime = self.am_getOption( 'MaxPilotWaitingTime', 3600 ) self.failedQueueCycleFactor = self.am_getOption( 'FailedQueueCycleFactor', 10 ) self.pilotStatusUpdateCycleFactor = self.am_getOption( 'PilotStatusUpdateCycleFactor', 10 ) # Flags self.updateStatus = self.am_getOption( 'UpdatePilotStatus', True ) self.getOutput = self.am_getOption( 'GetPilotOutput', True ) self.sendAccounting = self.am_getOption( 'SendPilotAccounting', True ) # Get the site description dictionary siteNames = None if not self.am_getOption( 'Site', 'Any' ).lower() == "any": siteNames = self.am_getOption( 'Site', [] ) if not siteNames: siteNames = None ceTypes = None if not self.am_getOption( 'CETypes', 'Any' ).lower() == "any": ceTypes = self.am_getOption( 'CETypes', [] ) ces = None if not self.am_getOption( 'CEs', 'Any' ).lower() == "any": ces = self.am_getOption( 'CEs', [] ) if not ces: ces = None result = Resources.getQueues( community = self.vo, siteList = siteNames, ceList = ces, ceTypeList = ceTypes, mode = 'Direct' ) if not result['OK']: return result resourceDict = result['Value'] result = self.getQueues( resourceDict ) if not result['OK']: return result #if not siteNames: # siteName = gConfig.getValue( '/DIRAC/Site', 'Unknown' ) # if siteName == 'Unknown': # return S_OK( 'No site specified for the SiteDirector' ) # else: # siteNames = [siteName] #self.siteNames = siteNames if self.updateStatus: self.log.always( 'Pilot status update requested' ) if self.getOutput: self.log.always( 'Pilot output retrieval requested' ) if self.sendAccounting: self.log.always( 'Pilot accounting sending requested' ) self.log.always( 'Sites:', siteNames ) self.log.always( 'CETypes:', ceTypes ) self.log.always( 'CEs:', ces ) self.log.always( 'PilotDN:', self.pilotDN ) self.log.always( 'PilotGroup:', self.pilotGroup ) self.log.always( 'MaxPilotsToSubmit:', self.maxPilotsToSubmit ) self.log.always( 'MaxJobsInFillMode:', self.maxJobsInFillMode ) self.localhost = socket.getfqdn() self.proxy = '' if self.firstPass: if self.queueDict: self.log.always( "Agent will serve queues:" ) for queue in self.queueDict: self.log.always( "Site: %s, CE: %s, Queue: %s" % ( self.queueDict[queue]['Site'], self.queueDict[queue]['CEName'], queue ) ) self.firstPass = False return S_OK()
def getProxyWithToken(self, token): """ Get proxy with token :param str token: access token :return: S_OK()/S_ERROR() """ # Get REST endpoints from local CS confUrl = gConfig.getValue("/LocalInstallation/ConfigurationServerAPI") if not confUrl: return S_ERROR('Could not get configuration server API URL.') setup = gConfig.getValue("/DIRAC/Setup") if not setup: return S_ERROR('Could not get setup name.') # Get REST endpoints from ConfigurationService try: r = requests.get( '%s/option?path=/Systems/Framework/Production/URLs/ProxyAPI' % confUrl, verify=False) r.raise_for_status() proxyAPI = decode(r.text)[0] except requests.exceptions.Timeout: return S_ERROR('Time out') except requests.exceptions.RequestException as e: return S_ERROR(str(e)) except Exception as e: return S_ERROR('Cannot read response: %s' % e) # Fill the proxy request URL url = '%ss:%s/g:%s/proxy?lifetime=%s' % (proxyAPI, setup, self.group, self.lifetime) voms = self.voms or Registry.getGroupOption(self.group, "AutoAddVOMS", False) if voms: url += '&voms=%s' % voms # Get proxy from REST API try: r = requests.get(url, headers={'Authorization': 'Bearer ' + token}, verify=False) r.raise_for_status() proxy = decode(r.text)[0] except requests.exceptions.Timeout: return S_ERROR('Time out') except requests.exceptions.RequestException as e: return S_ERROR(str(e)) except Exception as e: return S_ERROR('Cannot read response: %s' % e) if not proxy: return S_ERROR("Result is empty.") self.log.notice('Saving proxy.. to %s..' % self.pPath) # Save proxy to file try: with open(self.pPath, 'w+') as fd: fd.write(proxy.encode("UTF-8")) os.chmod(self.pPath, stat.S_IRUSR | stat.S_IWUSR) except Exception as e: return S_ERROR("%s :%s" % (self.pPath, repr(e).replace(',)', ')'))) self.log.notice('Proxy is saved to %s.' % self.pPath) return S_OK()