def _getBaseStub(self):
newKwargs = dict(self.kwargs)
#Set DN
if 'DN' in self.__idDict and not self.KW_DELEGATED_DN in newKwargs:
newKwargs[self.KW_DELEGATED_DN] = self.__idDict['DN']
#Discover group
if not self.KW_DELEGATED_GROUP in newKwargs:
if 'group' in self.__idDict:
newKwargs[self.KW_DELEGATED_GROUP] = self.__idDict['group']
else:
if self.KW_DELEGATED_DN in newKwargs:
if CS.getUsernameForDN(
newKwargs[self.KW_DELEGATED_DN])['OK']:
newKwargs[
self.KW_DELEGATED_GROUP] = CS.getDefaultUserGroup(
)
if CS.getHostnameForDN(
newKwargs[self.KW_DELEGATED_DN])['OK']:
newKwargs[
self.
KW_DELEGATED_GROUP] = self.VAL_EXTRA_CREDENTIALS_HOST
if 'useCertificates' in newKwargs:
del (newKwargs['useCertificates'])
return (self._destinationSrv, newKwargs)
def getCredentials( self ):
if not self.__loadedChain:
return S_ERROR( "No chain loaded" )
credDict = { 'subject' : self.__certList[0].get_subject().one_line(),
'issuer' : self.__certList[0].get_issuer().one_line(),
'secondsLeft' : self.getRemainingSecs()[ 'Value' ],
'isProxy' : self.__isProxy,
'isLimitedProxy' : self.__isProxy and self.__isLimitedProxy,
'validDN' : False,
'validGroup' : False }
if self.__isProxy:
retVal = self.getDIRACGroup()
if not retVal[ 'OK' ]:
return retVal
diracGroup = retVal[ 'Value' ]
if not diracGroup:
diracGroup = CS.getDefaultUserGroup()
credDict[ 'group' ] = diracGroup
credDict[ 'identity'] = self.__certList[ self.__firstProxyStep + 1 ].get_subject().one_line()
retVal = CS.getUsernameForDN( credDict[ 'identity' ] )
if retVal[ 'OK' ]:
credDict[ 'username' ] = retVal[ 'Value' ]
credDict[ 'validDN' ] = True
retVal = CS.getGroupsForUser( credDict[ 'username' ] )
if retVal[ 'OK' ] and diracGroup in retVal[ 'Value']:
credDict[ 'validGroup' ] = True
credDict[ 'groupProperties' ] = CS.getPropertiesForGroup( diracGroup )
else:
retVal = CS.getHostnameForDN( credDict['subject'] )
retVal[ 'group' ] = 'hosts'
if retVal[ 'OK' ]:
credDict[ 'hostname' ] = retVal[ 'Value' ]
credDict[ 'validDN' ] = True
credDict[ 'validGroup' ] = True
return S_OK( credDict )
def _getBaseStub( self ):
newKwargs = dict( self.kwargs )
#Set DN
tDN, tGroup = self.__threadConfig.getID()
if not self.KW_DELEGATED_DN in newKwargs:
if tDN:
newKwargs[ self.KW_DELEGATED_DN ] = tDN
elif 'DN' in self.__idDict:
newKwargs[ self.KW_DELEGATED_DN ] = self.__idDict[ 'DN' ]
#Discover group
if not self.KW_DELEGATED_GROUP in newKwargs:
if 'group' in self.__idDict:
newKwargs[ self.KW_DELEGATED_GROUP ] = self.__idDict[ 'group' ]
elif tGroup:
newKwargs[ self.KW_DELEGATED_GROUP ] = tGroup
else:
if self.KW_DELEGATED_DN in newKwargs:
if CS.getUsernameForDN( newKwargs[ self.KW_DELEGATED_DN ] )[ 'OK' ]:
result = CS.findDefaultGroupForDN( newKwargs[ self.KW_DELEGATED_DN ] )
if result['OK']:
newKwargs[ self.KW_DELEGATED_GROUP ] = result['Value']
if CS.getHostnameForDN( newKwargs[ self.KW_DELEGATED_DN ] )[ 'OK' ]:
newKwargs[ self.KW_DELEGATED_GROUP ] = self.VAL_EXTRA_CREDENTIALS_HOST
if 'useCertificates' in newKwargs:
del( newKwargs[ 'useCertificates' ] )
return ( self._destinationSrv, newKwargs )
def getUsername( self, credDict ):
"""
Discover the username associated to the DN. It will check if the selected group is valid.
The username will be included in the credentials dictionary.
:type credDict: dictionary
:param credDict: Credentials to ckeck
:return: Boolean specifying whether the username was found
"""
if self.KW_DN not in credDict:
return True
if self.KW_GROUP not in credDict:
result = CS.findDefaultGroupForDN( credDict[ self.KW_DN ] )
if not result['OK']:
return False
credDict[ self.KW_GROUP ] = result['Value']
credDict[ self.KW_PROPERTIES ] = CS.getPropertiesForGroup( credDict[ self.KW_GROUP ], [] )
usersInGroup = CS.getUsersInGroup( credDict[ self.KW_GROUP ], [] )
if not usersInGroup:
return False
retVal = CS.getUsernameForDN( credDict[ self.KW_DN ], usersInGroup )
if retVal[ 'OK' ]:
credDict[ self.KW_USERNAME ] = retVal[ 'Value' ]
return True
return False
def __getVOMSAttribute( self, userGroup, requiredVOMSAttribute = False ):
if requiredVOMSAttribute:
return S_OK( { 'attribute' : requiredVOMSAttribute, 'VOMSVO' : CS.getVOMSVOForGroup( userGroup ) } )
csVOMSMapping = CS.getVOMSAttributeForGroup( userGroup )
if not csVOMSMapping:
return S_ERROR( "No mapping defined for group %s in the CS" % userGroup )
return S_OK( { 'attribute' : csVOMSMapping, 'VOMSVO' : CS.getVOMSVOForGroup( userGroup ) } )
def getShifterProxy(shifterType, fileName=False):
"""
This method returns a shifter's proxy
:param shifterType: ProductionManager / DataManager...
"""
if fileName:
try:
os.makedirs(os.path.dirname(fileName))
except OSError:
pass
opsHelper = Operations()
userName = opsHelper.getValue(cfgPath('Shifter', shifterType, 'User'), '')
if not userName:
return S_ERROR("No shifter User defined for %s" % shifterType)
result = CS.getDNForUsername(userName)
if not result['OK']:
return result
userDN = result['Value'][0]
result = CS.findDefaultGroupForDN(userDN)
if not result['OK']:
return result
defaultGroup = result['Value']
userGroup = opsHelper.getValue(cfgPath('Shifter', shifterType, 'Group'),
defaultGroup)
vomsAttr = CS.getVOMSAttributeForGroup(userGroup)
if vomsAttr:
gLogger.info("Getting VOMS [%s] proxy for shifter %s@%s (%s)" %
(vomsAttr, userName, userGroup, userDN))
result = gProxyManager.downloadVOMSProxyToFile(userDN,
userGroup,
filePath=fileName,
requiredTimeLeft=86400,
cacheTime=86400)
else:
gLogger.info("Getting proxy for shifter %s@%s (%s)" %
(userName, userGroup, userDN))
result = gProxyManager.downloadProxyToFile(userDN,
userGroup,
filePath=fileName,
requiredTimeLeft=86400,
cacheTime=86400)
if not result['OK']:
return result
chain = result['chain']
fileName = result['Value']
return S_OK({
'DN': userDN,
'username': userName,
'group': userGroup,
'chain': chain,
'proxyFile': fileName
})
def __setupManagerProxies(self):
""" setup grid proxy for all defined managers """
oHelper = Operations()
shifters = oHelper.getSections("Shifter")
if not shifters["OK"]:
self.log.error(shifters["Message"])
return shifters
shifters = shifters["Value"]
for shifter in shifters:
shifterDict = oHelper.getOptionsDict("Shifter/%s" % shifter)
if not shifterDict["OK"]:
self.log.error(shifterDict["Message"])
continue
userName = shifterDict["Value"].get("User", "")
userGroup = shifterDict["Value"].get("Group", "")
userDN = CS.getDNForUsername(userName)
if not userDN["OK"]:
self.log.error(userDN["Message"])
continue
userDN = userDN["Value"][0]
vomsAttr = CS.getVOMSAttributeForGroup(userGroup)
if vomsAttr:
self.log.debug(
"getting VOMS [%s] proxy for shifter %s@%s (%s)" %
(vomsAttr, userName, userGroup, userDN))
getProxy = gProxyManager.downloadVOMSProxyToFile(
userDN,
userGroup,
requiredTimeLeft=1200,
cacheTime=4 * 43200)
else:
self.log.debug("getting proxy for shifter %s@%s (%s)" %
(userName, userGroup, userDN))
getProxy = gProxyManager.downloadProxyToFile(
userDN,
userGroup,
requiredTimeLeft=1200,
cacheTime=4 * 43200)
if not getProxy["OK"]:
self.log.error(getProxy["Message"])
return S_ERROR("unable to setup shifter proxy for %s: %s" %
(shifter, getProxy["Message"]))
chain = getProxy["chain"]
fileName = getProxy["Value"]
self.log.debug("got %s: %s %s" % (shifter, userName, userGroup))
self.__managersDict[shifter] = {
"ShifterDN": userDN,
"ShifterName": userName,
"ShifterGroup": userGroup,
"Chain": chain,
"ProxyFile": fileName
}
return S_OK()
def isTrustedHost(self, dn):
"""
Check if the request coming from a TrustedHost
:param str dn: certificate DN
:return: bool if the host is Trusrted it return true otherwise false
"""
retVal = CS.getHostnameForDN(dn)
if retVal['OK']:
hostname = retVal['Value']
if Properties.TRUSTED_HOST in CS.getPropertiesForHost(hostname, []):
return True
return False
def isTrustedHost(self, dn):
"""
Check if the request coming from a TrustedHost
:param str dn: certificate DN
:return: bool if the host is Trusrted it return true otherwise false
"""
retVal = CS.getHostnameForDN(dn)
if retVal['OK']:
hostname = retVal['Value']
if Properties.TRUSTED_HOST in CS.getPropertiesForHost(
hostname, []):
return True
return False
def completeDelegation(self, requestId, userDN, delegatedPem):
"""
Complete a delegation and store it in the db
"""
retVal = self.retrieveDelegationRequest(requestId, userDN)
if not retVal['OK']:
return retVal
request = retVal['Value']
chain = X509Chain(keyObj=request.getPKey())
retVal = chain.loadChainFromString(delegatedPem)
if not retVal['OK']:
return retVal
retVal = chain.isValidProxy()
if not retVal['OK']:
return retVal
if not retVal['Value']:
return S_ERROR("Chain received is not a valid proxy: %s" %
retVal['Message'])
retVal = request.checkChain(chain)
if not retVal['OK']:
return retVal
if not retVal['Value']:
return S_ERROR("Received chain does not match request: %s" %
retVal['Message'])
retVal = chain.getDIRACGroup()
if not retVal['OK']:
return retVal
userGroup = retVal['Value']
if not userGroup:
userGroup = CS.getDefaultUserGroup()
retVal = CS.getGroupsForDN(userDN)
if not retVal['OK']:
return retVal
if not userGroup in retVal['Value']:
return S_ERROR("%s group is not valid for %s" %
(userGroup, userDN))
retVal = self.__checkVOMSisAlignedWithGroup(userGroup, chain)
if not retVal['OK']:
return retVal
retVal = self.storeProxy(userDN, userGroup, chain)
if not retVal['OK']:
return retVal
retVal = self.deleteRequest(requestId)
if not retVal['OK']:
return retVal
return S_OK()
def getShifterProxy( shifterType, fileName = False ):
"""
This method returns a shifter's proxy
:param shifterType: ProductionManager / DataManager...
"""
if fileName:
try:
os.makedirs( os.path.dirname( fileName ) )
except OSError:
pass
opsHelper = Operations()
userName = opsHelper.getValue( cfgPath( 'Shifter', shifterType, 'User' ), '' )
if not userName:
return S_ERROR( "No shifter User defined for %s" % shifterType )
result = CS.getDNForUsername( userName )
if not result[ 'OK' ]:
return result
userDN = result[ 'Value' ][0]
result = CS.findDefaultGroupForDN( userDN )
if not result['OK']:
return result
defaultGroup = result['Value']
userGroup = opsHelper.getValue( cfgPath( 'Shifter', shifterType, 'Group' ), defaultGroup )
vomsAttr = CS.getVOMSAttributeForGroup( userGroup )
if vomsAttr:
gLogger.info( "Getting VOMS [%s] proxy for shifter %s@%s (%s)" % ( vomsAttr, userName,
userGroup, userDN ) )
result = gProxyManager.downloadVOMSProxyToFile( userDN, userGroup,
filePath = fileName,
requiredTimeLeft = 86400,
cacheTime = 86400 )
else:
gLogger.info( "Getting proxy for shifter %s@%s (%s)" % ( userName, userGroup, userDN ) )
result = gProxyManager.downloadProxyToFile( userDN, userGroup,
filePath = fileName,
requiredTimeLeft = 86400,
cacheTime = 86400 )
if not result[ 'OK' ]:
return result
chain = result[ 'chain' ]
fileName = result[ 'Value' ]
return S_OK( { 'DN' : userDN,
'username' : userName,
'group' : userGroup,
'chain' : chain,
'proxyFile' : fileName } )
def completeDelegation( self, requestId, userDN, delegatedPem ):
"""
Complete a delegation and store it in the db
"""
retVal = self.retrieveDelegationRequest( requestId, userDN )
if not retVal[ 'OK' ]:
return retVal
request = retVal[ 'Value' ]
chain = X509Chain( keyObj = request.getPKey() )
retVal = chain.loadChainFromString( delegatedPem )
if not retVal[ 'OK' ]:
return retVal
retVal = chain.isValidProxy()
if not retVal[ 'OK' ]:
return retVal
if not retVal[ 'Value' ]:
return S_ERROR( "Chain received is not a valid proxy: %s" % retVal[ 'Message' ] )
retVal = request.checkChain( chain )
if not retVal[ 'OK' ]:
return retVal
if not retVal[ 'Value' ]:
return S_ERROR( "Received chain does not match request: %s" % retVal[ 'Message' ] )
retVal = chain.getDIRACGroup()
if not retVal[ 'OK' ]:
return retVal
userGroup = retVal[ 'Value' ]
if not userGroup:
userGroup = CS.getDefaultUserGroup()
retVal = CS.getGroupsForDN( userDN )
if not retVal[ 'OK' ]:
return retVal
if not userGroup in retVal[ 'Value' ]:
return S_ERROR( "%s group is not valid for %s" % ( userGroup, userDN ) )
retVal = self.__checkVOMSisAlignedWithGroup( userGroup, chain )
if not retVal[ 'OK' ]:
return retVal
retVal = self.storeProxy( userDN, userGroup, chain )
if not retVal[ 'OK' ]:
return retVal
retVal = self.deleteRequest( requestId )
if not retVal[ 'OK' ]:
return retVal
return S_OK()
def __filterEntitiesByRequester( self, entitiesList, entitiesSetup, requesterName, requesterGroup ):
"""
Given a list of entities and a requester, return the ones that the requester is allowed to modify
"""
sqlCond = [ "s.OwnerId=o.OwnerId" , "s.SBId=e.SBId", "e.EntitySetup=%s" % entitiesSetup ]
requesterProps = CS.getPropertiesForEntity( requesterGroup, name = requesterName )
if Properties.JOB_ADMINISTRATOR in requesterProps:
#Do nothing, just ensure it doesn't fit in the other cases
pass
elif Properties.JOB_SHARING in requesterProps:
sqlCond.append( "o.OwnerGroup='%s'" % requesterGroup )
elif Properties.NORMAL_USER in requesterProps:
sqlCond.append( "o.OwnerGroup='%s'" % requesterGroup )
sqlCond.append( "o.Owner='%s'" % requesterName )
else:
return S_ERROR( "Not authorized to access sandbox" )
for i in range( len( entitiesList ) ):
entitiesList[i] = self._escapeString( entitiesList[ i ] )[ 'Value' ]
if len( entitiesList ) == 1:
sqlCond.append( "e.EntityId = %s" % entitiesList[0] )
else:
sqlCond.append( "e.EntityId in ( %s )" % ", ".join( entitiesList ) )
sqlCmd = "SELECT DISTINCT e.EntityId FROM `sb_EntityMapping` e, `sb_SandBoxes` s, `sb_Owners` o WHERE"
sqlCmd = "%s %s" % ( sqlCmd, " AND ".join( sqlCond ) )
result = self._query( sqlCmd )
if not result[ 'OK' ]:
return result
return S_OK( [ row[0] for row in result[ 'Value' ] ] )
def __discoverCredentialsToUse( self ):
""" Discovers which credentials to use for connection.
* Server certificate:
-> If KW_USE_CERTIFICATES in kwargs, sets it in self.__useCertificates
->If not, check gConfig.useServerCertificate(), and sets it in self.__useCertificates and kwargs[KW_USE_CERTIFICATES]
* Certification Authorities check:
-> if KW_SKIP_CA_CHECK is not in kwargs and we are using the certificates, set KW_SKIP_CA_CHECK to false in kwargs
-> if KW_SKIP_CA_CHECK is not in kwargs and we are not using the certificate, check the CS.skipCACheck
* Proxy Chain
-> if KW_PROXY_CHAIN in kwargs, we remove it and dump its string form into kwargs[KW_PROXY_STRING]
"""
#Use certificates?
if self.KW_USE_CERTIFICATES in self.kwargs:
self.__useCertificates = self.kwargs[ self.KW_USE_CERTIFICATES ]
else:
self.__useCertificates = gConfig.useServerCertificate()
self.kwargs[ self.KW_USE_CERTIFICATES ] = self.__useCertificates
if self.KW_SKIP_CA_CHECK not in self.kwargs:
if self.__useCertificates:
self.kwargs[ self.KW_SKIP_CA_CHECK ] = False
else:
self.kwargs[ self.KW_SKIP_CA_CHECK ] = CS.skipCACheck()
if self.KW_PROXY_CHAIN in self.kwargs:
try:
self.kwargs[ self.KW_PROXY_STRING ] = self.kwargs[ self.KW_PROXY_CHAIN ].dumpAllToString()[ 'Value' ]
del self.kwargs[ self.KW_PROXY_CHAIN ]
except:
return S_ERROR( "Invalid proxy chain specified on instantiation" )
return S_OK()
def __discoverCredentialsToUse(self):
""" Discovers which credentials to use for connection.
* Server certificate:
-> If KW_USE_CERTIFICATES in kwargs, sets it in self.__useCertificates
->If not, check gConfig.useServerCertificate(), and sets it in self.__useCertificates and kwargs[KW_USE_CERTIFICATES]
* Certification Authorities check:
-> if KW_SKIP_CA_CHECK is not in kwargs and we are using the certificates, set KW_SKIP_CA_CHECK to false in kwargs
-> if KW_SKIP_CA_CHECK is not in kwargs and we are not using the certificate, check the CS.skipCACheck
* Proxy Chain
-> if KW_PROXY_CHAIN in kwargs, we remove it and dump its string form into kwargs[KW_PROXY_STRING]
"""
# Use certificates?
if self.KW_USE_CERTIFICATES in self.kwargs:
self.__useCertificates = self.kwargs[self.KW_USE_CERTIFICATES]
else:
self.__useCertificates = gConfig.useServerCertificate()
self.kwargs[self.KW_USE_CERTIFICATES] = self.__useCertificates
if self.KW_SKIP_CA_CHECK not in self.kwargs:
if self.__useCertificates:
self.kwargs[self.KW_SKIP_CA_CHECK] = False
else:
self.kwargs[self.KW_SKIP_CA_CHECK] = CS.skipCACheck()
if self.KW_PROXY_CHAIN in self.kwargs:
try:
self.kwargs[self.KW_PROXY_STRING] = self.kwargs[self.KW_PROXY_CHAIN].dumpAllToString()['Value']
del self.kwargs[self.KW_PROXY_CHAIN]
except BaseException:
return S_ERROR("Invalid proxy chain specified on instantiation")
return S_OK()
def __addPilotsAccountingReport( self, pilotsData ):
""" fill accounting data
"""
for pRef in pilotsData:
pData = pilotsData[pRef]
pA = PilotAccounting()
pA.setEndTime( pData[ 'LastUpdateTime' ] )
pA.setStartTime( pData[ 'SubmissionTime' ] )
retVal = CS.getUsernameForDN( pData[ 'OwnerDN' ] )
if not retVal[ 'OK' ]:
userName = '******'
self.log.error( "Can't determine username for dn:", pData[ 'OwnerDN' ] )
else:
userName = retVal[ 'Value' ]
pA.setValueByKey( 'User', userName )
pA.setValueByKey( 'UserGroup', pData[ 'OwnerGroup' ] )
result = getSiteForCE( pData[ 'DestinationSite' ] )
if result['OK'] and result[ 'Value' ].strip():
pA.setValueByKey( 'Site', result['Value'].strip() )
else:
pA.setValueByKey( 'Site', 'Unknown' )
pA.setValueByKey( 'GridCE', pData[ 'DestinationSite' ] )
pA.setValueByKey( 'GridMiddleware', pData[ 'GridType' ] )
pA.setValueByKey( 'GridResourceBroker', pData[ 'Broker' ] )
pA.setValueByKey( 'GridStatus', pData[ 'Status' ] )
if not 'Jobs' in pData:
pA.setValueByKey( 'Jobs', 0 )
else:
pA.setValueByKey( 'Jobs', len( pData['Jobs'] ) )
self.log.verbose( "Added accounting record for pilot %s" % pData[ 'PilotID' ] )
retVal = gDataStoreClient.addRegister( pA )
if not retVal[ 'OK' ]:
return retVal
return S_OK()
def export_requestDelegationUpload( self, requestedUploadTime, userGroup ):
""" Request a delegation. Send a delegation request to client
"""
credDict = self.getRemoteCredentials()
userDN = credDict[ 'DN' ]
userName = credDict[ 'username' ]
if not userGroup:
userGroup = credDict[ 'group' ]
retVal = CS.getGroupsForUser( credDict[ 'username' ] )
if not retVal[ 'OK' ]:
return retVal
groupsAvailable = retVal[ 'Value' ]
if userGroup not in groupsAvailable:
return S_ERROR( "%s is not a valid group for user %s" % ( userGroup, userName ) )
retVal = gProxyDB.getRemainingTime( userDN, userGroup )
if not retVal[ 'OK' ]:
return retVal
remainingSecs = retVal[ 'Value' ]
csOption = "%s/SkipUploadLifeTime" % self.serviceInfoDict[ 'serviceSectionPath' ]
#If we have a proxy longer than the one uploading it's not needed
if remainingSecs > requestedUploadTime:
gLogger.info( "Upload request not necessary by %s:%s" % ( userName, userGroup ) )
return S_OK()
result = gProxyDB.generateDelegationRequest( credDict[ 'x509Chain' ], userDN )
if result[ 'OK' ]:
gLogger.info( "Upload request by %s:%s given id %s" % ( userName, userGroup, result['Value']['id'] ) )
else:
gLogger.error( "Upload request failed", "by %s:%s : %s" % ( userName, userGroup, result['Message'] ) )
return result
def addDataset(self, name, datasetDict, numericid):
""" Pretty print of the dataset ls output
"""
perm = datasetDict["Mode"]
date = datasetDict["ModificationDate"]
size = datasetDict["TotalSize"]
if datasetDict.has_key("Owner"):
uname = datasetDict["Owner"]
elif datasetDict.has_key("OwnerDN"):
result = CS.getUsernameForDN(datasetDict["OwnerDN"])
if result["OK"]:
uname = result["Value"]
else:
uname = "unknown"
else:
uname = "unknown"
if numericid:
uname = str(datasetDict["UID"])
gname = "unknown"
if datasetDict.has_key("OwnerGroup"):
gname = datasetDict["OwnerGroup"]
if numericid:
gname = str(datasetDict["GID"])
numberOfFiles = datasetDict["NumberOfFiles"]
self.entries.append(("s" + self.__getModeString(perm), numberOfFiles, uname, gname, size, date, name))
def checkInputSandbox( self, job, classAdJob ):
"""The number of input sandbox files, as specified in the job
JDL are checked in the JobDB.
"""
ownerName = classAdJob.getAttributeString( "Owner" )
if not ownerName:
ownerDN = classAdJob.getAttributeString( "OwnerDN" )
ownerName = CS.getUsernameForDN( ownerDN )
ownerGroup = classAdJob.getAttributeString( "OwnerGroup" )
jobSetup = classAdJob.getAttributeString( "DIRACSetup" )
isbList = classAdJob.getListFromExpression( 'InputSandbox' )
sbsToAssign = []
for isb in isbList:
if isb.find( "SB:" ) == 0:
self.log.info( "Found a sandbox", isb )
sbsToAssign.append( ( isb, "Input" ) )
numSBsToAssign = len( sbsToAssign )
if not numSBsToAssign:
return S_OK( 0 )
self.log.info( "Assigning %s sandboxes on behalf of %s@%s" % ( numSBsToAssign, ownerName, ownerGroup ) )
result = self.sandboxClient.assignSandboxesToJob( job, sbsToAssign, ownerName, ownerGroup, jobSetup )
if not result[ 'OK' ]:
self.log.error( "Could not assign sandboxes in the SandboxStore", "assigned to job %s" % job )
return S_ERROR( "Cannot assign sandbox to job" )
assigned = result[ 'Value' ]
if assigned != numSBsToAssign:
self.log.error( "Could not assign all sandboxes (%s). Only assigned %s" % ( numSBsToAssign, assigned ) )
return S_OK( numSBsToAssign )
def getSandboxesAssignedToEntity( self, entityId, entitySetup, requesterName, requesterGroup ):
"""
Get the sandboxes and the type of assignation to the jobId
"""
sqlTables = [ "`sb_SandBoxes` s", "`sb_EntityMapping` e" ]
sqlCond = [ "s.SBId = e.SBId",
"e.EntityId = %s" % self._escapeString( entityId )[ 'Value' ],
"e.EntitySetup = %s" % self._escapeString( entitySetup )[ 'Value' ] ]
requesterProps = CS.getPropertiesForEntity( requesterGroup, name = requesterName )
if Properties.JOB_ADMINISTRATOR in requesterProps:
#Do nothing, just ensure it doesn't fit in the other cases
pass
elif Properties.JOB_SHARING in requesterProps:
sqlTables.append( "`sb_Owners` o" )
sqlCond.append( "o.OwnerGroup='%s'" % requesterGroup )
sqlCond.append( "s.OwnerId=o.OwnerId" )
elif Properties.NORMAL_USER in requesterProps:
sqlTables.append( "`sb_Owners` o" )
sqlCond.append( "o.OwnerGroup='%s'" % requesterGroup )
sqlCond.append( "o.Owner='%s'" % requesterName )
sqlCond.append( "s.OwnerId=o.OwnerId" )
else:
return S_ERROR( "Not authorized to access sandbox" )
sqlCmd = "SELECT DISTINCT s.SEName, s.SEPFN, e.Type FROM %s WHERE %s" % ( ", ".join( sqlTables ),
" AND ".join( sqlCond ) )
return self._query( sqlCmd )
def __checkDN(environ):
userDN = False
if "SERVER_SOFTWARE" not in environ:
diracLogger.info("Getting the DN from /Website/DebugDN")
userDN = gWebConfig.getDebugDN()
if "HTTPS" in environ and environ["HTTPS"] == "on":
if "SSL_CLIENT_S_DN" in environ:
userDN = environ["SSL_CLIENT_S_DN"]
elif "SSL_CLIENT_CERT" in environ:
userCert = X509Certificate.X509Certificate()
result = userCert.loadFromString(environ["SSL_CLIENT_CERT"])
if not result["OK"]:
diracLogger.error("Could not load SSL_CLIENT_CERT: %s" % result["Message"])
userName = "******"
else:
userDN = userCert.getSubjectDN()["Value"]
else:
diracLogger.error("Web server is not properly configured to get SSL_CLIENT_S_DN or SSL_CLIENT_CERT in env")
if not userDN:
userName = "******"
else:
retVal = CS.getUsernameForDN(userDN)
if not retVal["OK"]:
userName = "******"
else:
userName = retVal["Value"]
diracLogger.info("Got username for user" " => %s for %s" % (userName, userDN))
return (userDN, userName)
def checkInputSandbox( self, job, classAdJob ):
"""The number of input sandbox files, as specified in the job
JDL are checked in the JobDB.
"""
ownerName = classAdJob.getAttributeString( "Owner" )
if not ownerName:
ownerDN = classAdJob.getAttributeString( "OwnerDN" )
ownerName = CS.getUsernameForDN( ownerDN )
ownerGroup = classAdJob.getAttributeString( "OwnerGroup" )
jobSetup = classAdJob.getAttributeString( "DIRACSetup" )
isbList = classAdJob.getListFromExpression( 'InputSandbox' )
sbsToAssign = []
for isb in isbList:
if isb.find( "SB:" ) == 0:
self.log.info( "Found a sandbox", isb )
sbsToAssign.append( ( isb, "Input" ) )
numSBsToAssign = len( sbsToAssign )
if not numSBsToAssign:
return S_OK( 0 )
self.log.info( "Assigning %s sandboxes on behalf of %s@%s" % ( numSBsToAssign, ownerName, ownerGroup ) )
result = self.sandboxClient.assignSandboxesToJob( job, sbsToAssign, ownerName, ownerGroup, jobSetup )
if not result[ 'OK' ]:
self.log.error( "Could not assign sandboxes in the SandboxStore", "assigned to job %s" % job )
return S_ERROR( "Cannot assign sandbox to job" )
assigned = result[ 'Value' ]
if assigned != numSBsToAssign:
self.log.error( "Could not assign all sandboxes (%s). Only assigned %s" % ( numSBsToAssign, assigned ) )
return S_OK( numSBsToAssign )
def getSandboxId( self, SEName, SEPFN, requesterName, requesterGroup ):
"""
Get the sandboxId if it exists
"""
sqlCond = [ "s.SEPFN=%s" % self._escapeString( SEPFN )['Value'],
"s.SEName=%s" % self._escapeString( SEName )['Value'],
's.OwnerId=o.OwnerId' ]
sqlCmd = "SELECT s.SBId FROM `sb_SandBoxes` s, `sb_Owners` o WHERE"
requesterProps = CS.getPropertiesForEntity( requesterGroup, name = requesterName )
if Properties.JOB_ADMINISTRATOR in requesterProps:
#Do nothing, just ensure it doesn't fit in the other cases
pass
elif Properties.JOB_SHARING in requesterProps:
sqlCond.append( "o.OwnerGroup='%s'" % requesterGroup )
elif Properties.NORMAL_USER in requesterProps:
sqlCond.append( "o.OwnerGroup='%s'" % requesterGroup )
sqlCond.append( "o.Owner='%s'" % requesterName )
else:
return S_ERROR( "Not authorized to access sandbox" )
result = self._query( "%s %s" % ( sqlCmd, " AND ".join( sqlCond ) ) )
if not result[ 'OK' ]:
return result
data = result[ 'Value' ]
if len( data ) > 1:
self.log.error( "More than one sandbox registered with the same Id!", data )
if len( data ) == 0:
return S_ERROR( "No sandbox matches the requirements" )
return S_OK( data[0][0] )
def addDataset( self, name, datasetDict, numericid ):
""" Pretty print of the dataset ls output
"""
perm = datasetDict['Mode']
date = datasetDict['ModificationDate']
size = datasetDict['TotalSize']
if datasetDict.has_key('Owner'):
uname = datasetDict['Owner']
elif datasetDict.has_key('OwnerDN'):
result = CS.getUsernameForDN(datasetDict['OwnerDN'])
if result['OK']:
uname = result['Value']
else:
uname = 'unknown'
else:
uname = 'unknown'
if numericid:
uname = str( datasetDict['UID'] )
gname = 'unknown'
if datasetDict.has_key('OwnerGroup'):
gname = datasetDict['OwnerGroup']
if numericid:
gname = str( datasetDict ['GID'] )
numberOfFiles = datasetDict ['NumberOfFiles']
self.entries.append( ('s'+self.__getModeString(perm),numberOfFiles,uname,gname,size,date,name) )
def __checkDN(environ):
userDN = False
if 'SERVER_SOFTWARE' not in environ:
diracLogger.info("Getting the DN from /Website/DebugDN")
userDN = gWebConfig.getDebugDN()
if 'HTTPS' in environ and environ['HTTPS'] == 'on':
if 'SSL_CLIENT_S_DN' in environ:
userDN = environ['SSL_CLIENT_S_DN']
elif 'SSL_CLIENT_CERT' in environ:
userCert = X509Certificate.X509Certificate()
result = userCert.loadFromString(environ['SSL_CLIENT_CERT'])
if not result['OK']:
diracLogger.error("Could not load SSL_CLIENT_CERT: %s" %
result['Message'])
userName = "******"
else:
userDN = userCert.getSubjectDN()['Value']
else:
diracLogger.error(
"Web server is not properly configured to get SSL_CLIENT_S_DN or SSL_CLIENT_CERT in env"
)
if not userDN:
userName = "******"
else:
retVal = CS.getUsernameForDN(userDN)
if not retVal['OK']:
userName = "******"
else:
userName = retVal['Value']
diracLogger.info("Got username for user"
" => %s for %s" % (userName, userDN))
return (userDN, userName)
def __getUniqueKeyValues(self, typeName):
sessionData = SessionData().getData()
userGroup = sessionData["user"]["group"]
if 'NormalUser' in CS.getPropertiesForGroup(userGroup):
cacheKey = (sessionData["user"]["username"], userGroup,
sessionData["setup"], typeName)
else:
cacheKey = (userGroup, sessionData["setup"], typeName)
data = AccountingPlotOldHandler.__keysCache.get(cacheKey)
if not data:
rpcClient = RPCClient("Accounting/ReportGenerator")
retVal = rpcClient.listUniqueKeyValues(typeName)
if 'rpcStub' in retVal:
del (retVal['rpcStub'])
if not retVal['OK']:
return retVal
#Site ordering based on TierLevel / alpha
if 'Site' in retVal['Value']:
siteLevel = {}
for siteName in retVal['Value']['Site']:
sitePrefix = siteName.split(".")[0].strip()
level = gConfig.getValue(
"/Resources/Sites/%s/%s/MoUTierLevel" %
(sitePrefix, siteName), 10)
if level not in siteLevel:
siteLevel[level] = []
siteLevel[level].append(siteName)
orderedSites = []
for level in sorted(siteLevel):
orderedSites.extend(sorted(siteLevel[level]))
retVal['Value']['Site'] = orderedSites
data = retVal
AccountingPlotOldHandler.__keysCache.add(cacheKey, 300, data)
return data
def __addPilotsAccountingReport(self, pilotsData):
""" fill accounting data
"""
for pRef in pilotsData:
pData = pilotsData[pRef]
pA = PilotAccounting()
pA.setEndTime(pData["LastUpdateTime"])
pA.setStartTime(pData["SubmissionTime"])
retVal = CS.getUsernameForDN(pData["OwnerDN"])
if not retVal["OK"]:
userName = "******"
self.log.error("Can't determine username for dn:", pData["OwnerDN"])
else:
userName = retVal["Value"]
pA.setValueByKey("User", userName)
pA.setValueByKey("UserGroup", pData["OwnerGroup"])
result = getSiteForCE(pData["DestinationSite"])
if result["OK"] and result["Value"].strip():
pA.setValueByKey("Site", result["Value"].strip())
else:
pA.setValueByKey("Site", "Unknown")
pA.setValueByKey("GridCE", pData["DestinationSite"])
pA.setValueByKey("GridMiddleware", pData["GridType"])
pA.setValueByKey("GridResourceBroker", pData["Broker"])
pA.setValueByKey("GridStatus", pData["Status"])
if not "Jobs" in pData:
pA.setValueByKey("Jobs", 0)
else:
pA.setValueByKey("Jobs", len(pData["Jobs"]))
self.log.verbose("Added accounting record for pilot %s" % pData["PilotID"])
retVal = gDataStoreClient.addRegister(pA)
if not retVal["OK"]:
return retVal
return S_OK()
def _getUsageHistoryForTimeSpan( self, timeSpan, groupToUse = "" ):
reportCondition = { 'Status' : [ 'Running' ] }
if not groupToUse:
reportGrouping = 'UserGroup'
else:
reportGrouping = 'User'
reportCondition = { 'UserGroup' : groupToUse }
now = Time.dateTime()
result = self.__reportsClient.getReport( 'WMSHistory', 'AverageNumberOfJobs',
now - datetime.timedelta( seconds = timeSpan ), now,
reportCondition, reportGrouping,
{ 'lastSeconds' : timeSpan } )
if not result[ 'OK' ]:
self.__log.error( "Cannot get history from Accounting", result[ 'Message' ] )
return result
data = result['Value'].get( 'data', [] )
if not data:
message = "Empty history data from Accounting"
self.__log.error( message )
return S_ERROR( message )
#Map the usernames to DNs
if groupToUse:
mappedData = {}
for userName in data:
result = CS.getDNForUsername( userName )
if not result[ 'OK' ]:
self.__log.error( "User does not have any DN assigned", "%s :%s" % ( userName, result[ 'Message' ] ) )
continue
for userDN in result[ 'Value' ]:
mappedData[ userDN ] = data[ userName ]
data = mappedData
return S_OK( data )
def getSandboxesAssignedToEntity( self, entityId, entitySetup, requesterName, requesterGroup ):
"""
Get the sandboxes and the type of assignation to the jobId
"""
sqlTables = [ "`sb_SandBoxes` s", "`sb_EntityMapping` e" ]
sqlCond = [ "s.SBId = e.SBId",
"e.EntityId = %s" % self._escapeString( entityId )[ 'Value' ],
"e.EntitySetup = %s" % self._escapeString( entitySetup )[ 'Value' ] ]
requesterProps = CS.getPropertiesForEntity( requesterGroup, name = requesterName )
if Properties.JOB_ADMINISTRATOR in requesterProps or Properties.JOB_MONITOR in requesterProps:
#Do nothing, just ensure it doesn't fit in the other cases
pass
elif Properties.JOB_SHARING in requesterProps:
sqlTables.append( "`sb_Owners` o" )
sqlCond.append( "o.OwnerGroup='%s'" % requesterGroup )
sqlCond.append( "s.OwnerId=o.OwnerId" )
elif Properties.NORMAL_USER in requesterProps:
sqlTables.append( "`sb_Owners` o" )
sqlCond.append( "o.OwnerGroup='%s'" % requesterGroup )
sqlCond.append( "o.Owner='%s'" % requesterName )
sqlCond.append( "s.OwnerId=o.OwnerId" )
else:
return S_ERROR( "Not authorized to access sandbox" )
sqlCmd = "SELECT DISTINCT s.SEName, s.SEPFN, e.Type FROM %s WHERE %s" % ( ", ".join( sqlTables ),
" AND ".join( sqlCond ) )
return self._query( sqlCmd )
def getSandboxId( self, SEName, SEPFN, requesterName, requesterGroup ):
"""
Get the sandboxId if it exists
"""
sqlCond = [ "s.SEPFN=%s" % self._escapeString( SEPFN )['Value'],
"s.SEName=%s" % self._escapeString( SEName )['Value'],
's.OwnerId=o.OwnerId' ]
sqlCmd = "SELECT s.SBId FROM `sb_SandBoxes` s, `sb_Owners` o WHERE"
requesterProps = CS.getPropertiesForEntity( requesterGroup, name = requesterName )
if Properties.JOB_ADMINISTRATOR in requesterProps or Properties.JOB_MONITOR in requesterProps:
#Do nothing, just ensure it doesn't fit in the other cases
pass
elif Properties.JOB_SHARING in requesterProps:
sqlCond.append( "o.OwnerGroup='%s'" % requesterGroup )
elif Properties.NORMAL_USER in requesterProps:
sqlCond.append( "o.OwnerGroup='%s'" % requesterGroup )
sqlCond.append( "o.Owner='%s'" % requesterName )
else:
return S_ERROR( "Not authorized to access sandbox" )
result = self._query( "%s %s" % ( sqlCmd, " AND ".join( sqlCond ) ) )
if not result[ 'OK' ]:
return result
data = result[ 'Value' ]
if len( data ) > 1:
self.log.error( "More than one sandbox registered with the same Id!", data )
if len( data ) == 0:
return S_ERROR( "No sandbox matches the requirements" )
return S_OK( data[0][0] )
def __filterEntitiesByRequester( self, entitiesList, entitiesSetup, requesterName, requesterGroup ):
"""
Given a list of entities and a requester, return the ones that the requester is allowed to modify
"""
sqlCond = [ "s.OwnerId=o.OwnerId" , "s.SBId=e.SBId", "e.EntitySetup=%s" % entitiesSetup ]
requesterProps = CS.getPropertiesForEntity( requesterGroup, name = requesterName )
if Properties.JOB_ADMINISTRATOR in requesterProps:
#Do nothing, just ensure it doesn't fit in the other cases
pass
elif Properties.JOB_SHARING in requesterProps:
sqlCond.append( "o.OwnerGroup='%s'" % requesterGroup )
elif Properties.NORMAL_USER in requesterProps:
sqlCond.append( "o.OwnerGroup='%s'" % requesterGroup )
sqlCond.append( "o.Owner='%s'" % requesterName )
else:
return S_ERROR( "Not authorized to access sandbox" )
for i in range( len( entitiesList ) ):
entitiesList[i] = self._escapeString( entitiesList[ i ] )[ 'Value' ]
if len( entitiesList ) == 1:
sqlCond.append( "e.EntityId = %s" % entitiesList[0] )
else:
sqlCond.append( "e.EntityId in ( %s )" % ", ".join( entitiesList ) )
sqlCmd = "SELECT DISTINCT e.EntityId FROM `sb_EntityMapping` e, `sb_SandBoxes` s, `sb_Owners` o WHERE"
sqlCmd = "%s %s" % ( sqlCmd, " AND ".join( sqlCond ) )
result = self._query( sqlCmd )
if not result[ 'OK' ]:
return result
return S_OK( [ row[0] for row in result[ 'Value' ] ] )
def forwardedCredentials( self, credDict ):
"""
Check whether the credentials are being forwarded by a valid source
:type credDict: dictionary
:param credDict: Credentials to ckeck
:return: Boolean with the result
"""
if self.KW_EXTRA_CREDENTIALS in credDict and type( credDict[ self.KW_EXTRA_CREDENTIALS ] ) == types.TupleType:
if self.KW_DN in credDict:
retVal = CS.getHostnameForDN( credDict[ self.KW_DN ] )
if retVal[ 'OK' ]:
hostname = retVal[ 'Value' ]
if Properties.TRUSTED_HOST in CS.getPropertiesForHost( hostname, [] ):
return True
return False
def getPayloadProxyFromDIRACGroup(self, userDN, userGroup, requiredTimeLeft, token=False, proxyToConnect=False):
"""
Download a payload proxy with VOMS extensions depending on the group
"""
# Assign VOMS attribute
vomsAttr = CS.getVOMSAttributeForGroup(userGroup)
if not vomsAttr:
gLogger.verbose("No voms attribute assigned to group %s when requested payload proxy" % userGroup)
return self.downloadProxy(
userDN,
userGroup,
limited=True,
requiredTimeLeft=requiredTimeLeft,
proxyToConnect=proxyToConnect,
token=token,
)
else:
return self.downloadVOMSProxy(
userDN,
userGroup,
limited=True,
requiredTimeLeft=requiredTimeLeft,
requiredVOMSAttribute=vomsAttr,
proxyToConnect=proxyToConnect,
token=token,
)
def __addPilotsAccountingReport(self, pilotsData):
""" fill accounting data
"""
for pRef in pilotsData:
pData = pilotsData[pRef]
pA = PilotAccounting()
pA.setEndTime(pData['LastUpdateTime'])
pA.setStartTime(pData['SubmissionTime'])
retVal = CS.getUsernameForDN(pData['OwnerDN'])
if not retVal['OK']:
userName = '******'
self.log.error("Can't determine username for dn:", pData['OwnerDN'])
else:
userName = retVal['Value']
pA.setValueByKey('User', userName)
pA.setValueByKey('UserGroup', pData['OwnerGroup'])
result = getSiteForCE(pData['DestinationSite'])
if result['OK'] and result['Value'].strip():
pA.setValueByKey('Site', result['Value'].strip())
else:
pA.setValueByKey('Site', 'Unknown')
pA.setValueByKey('GridCE', pData['DestinationSite'])
pA.setValueByKey('GridMiddleware', pData['GridType'])
pA.setValueByKey('GridResourceBroker', pData['Broker'])
pA.setValueByKey('GridStatus', pData['Status'])
if 'Jobs' not in pData:
pA.setValueByKey('Jobs', 0)
else:
pA.setValueByKey('Jobs', len(pData['Jobs']))
self.log.verbose("Added accounting record for pilot %s" % pData['PilotID'])
retVal = gDataStoreClient.addRegister(pA)
if not retVal['OK']:
return retVal
return S_OK()
def __getUniqueKeyValues( self, typeName ):
sessionData = SessionData().getData()
userGroup = sessionData["user"]["group"]
if 'NormalUser' in CS.getPropertiesForGroup( userGroup ):
cacheKey = ( sessionData["user"]["username"], userGroup, sessionData["setup"], typeName )
else:
cacheKey = ( userGroup, sessionData["setup"], typeName )
data = AccountingPlotHandler.__keysCache.get( cacheKey )
if not data:
rpcClient = RPCClient( "Accounting/ReportGenerator" )
retVal = rpcClient.listUniqueKeyValues( typeName )
if 'rpcStub' in retVal:
del( retVal[ 'rpcStub' ] )
if not retVal[ 'OK' ]:
return retVal
#Site ordering based on TierLevel / alpha
if 'Site' in retVal[ 'Value' ]:
siteLevel = {}
for siteName in retVal[ 'Value' ][ 'Site' ]:
sitePrefix = siteName.split( "." )[0].strip()
level = gConfig.getValue( "/Resources/Sites/%s/%s/MoUTierLevel" % ( sitePrefix, siteName ), 10 )
if level not in siteLevel:
siteLevel[ level ] = []
siteLevel[ level ].append( siteName )
orderedSites = []
for level in sorted( siteLevel ):
orderedSites.extend( sorted( siteLevel[ level ] ) )
retVal[ 'Value' ][ 'Site' ] = orderedSites
data = retVal
AccountingPlotHandler.__keysCache.add( cacheKey, 300, data )
return data
def addNotificationForUser(self, user, message, lifetime=0, deferToMail=1):
if user not in CS.getAllUsers():
return S_ERROR("%s is an unknown user" % user)
self.log.info("Adding a notification for user %s (msg is %s chars)" %
(user, len(message)))
result = self._escapeString(user)
if not result['OK']:
return result
user = result['Value']
result = self._escapeString(message)
if not result['OK']:
return result
message = result['Value']
sqlFields = ['User', 'Message', 'Timestamp']
sqlValues = [user, message, 'UTC_TIMESTAMP()']
if not deferToMail:
sqlFields.append("DeferToMail")
sqlValues.append("0")
if lifetime:
sqlFields.append("Expiration")
sqlValues.append("TIMESTAMPADD( SECOND, %d, UTC_TIMESTAMP() )" %
int(lifetime))
sqlInsert = "INSERT INTO `ntf_Notifications` (%s) VALUES (%s) " % (
",".join(sqlFields), ",".join(sqlValues))
result = self._update(sqlInsert)
if not result['OK']:
return result
return S_OK(result['lastRowId'])
def _getUsageHistoryForTimeSpan(self, timeSpan, groupToUse=""):
reportCondition = {'Status': ['Running']}
if not groupToUse:
reportGrouping = 'UserGroup'
else:
reportGrouping = 'User'
reportCondition = {'UserGroup': groupToUse}
now = Time.dateTime()
result = self.__reportsClient.getReport(
'WMSHistory', 'AverageNumberOfJobs',
now - datetime.timedelta(seconds=timeSpan), now, reportCondition,
reportGrouping, {'lastSeconds': timeSpan})
if not result['OK']:
self.__log.error("Cannot get history from Accounting",
result['Message'])
return result
data = result['Value']['data']
#Map the usernames to DNs
if groupToUse:
mappedData = {}
for userName in data:
result = CS.getDNForUsername(userName)
if not result['OK']:
self.__log.error("User does not have any DN assigned",
"%s :%s" % (userName, result['Message']))
continue
for userDN in result['Value']:
mappedData[userDN] = data[userName]
data = mappedData
return S_OK(data)
def __discoverExtraCredentials( self ):
#Wich extra credentials to use?
if self.useCertificates:
self.__extraCredentials = self.VAL_EXTRA_CREDENTIALS_HOST
else:
self.__extraCredentials = ""
if self.KW_EXTRA_CREDENTIALS in self.kwargs:
self.__extraCredentials = self.kwargs[ self.KW_EXTRA_CREDENTIALS ]
#Are we delegating something?
delegatedDN, delegatedGroup = self.__threadConfig.getID()
if self.KW_DELEGATED_DN in self.kwargs and self.kwargs[ self.KW_DELEGATED_DN ]:
delegatedDN = self.kwargs[ self.KW_DELEGATED_DN ]
elif delegatedDN:
self.kwargs[ self.KW_DELEGATED_DN ] = delegatedDN
if self.KW_DELEGATED_GROUP in self.kwargs and self.kwargs[ self.KW_DELEGATED_GROUP ]:
delegatedGroup = self.kwargs[ self.KW_DELEGATED_GROUP ]
elif delegatedGroup:
self.kwargs[ self.KW_DELEGATED_GROUP ] = delegatedGroup
if delegatedDN:
if not delegatedGroup:
result = CS.findDefaultGroupForDN( self.kwargs[ self.KW_DELEGATED_DN ] )
if not result['OK']:
return result
self.__extraCredentials = ( delegatedDN, delegatedGroup )
return S_OK()
def addNotificationForUser( self, user, message, lifetime = 0, deferToMail = 1 ):
if user not in CS.getAllUsers():
return S_ERROR( "%s is an unknown user" % user )
self.log.info( "Adding a notification for user %s (msg is %s chars)" % ( user, len( message ) ) )
result = self._escapeString( user )
if not result[ 'OK' ]:
return result
user = result[ 'Value' ]
result = self._escapeString( message )
if not result[ 'OK' ]:
return result
message = result[ 'Value' ]
sqlFields = [ 'User', 'Message', 'Timestamp' ]
sqlValues = [ user, message, 'UTC_TIMESTAMP()' ]
if not deferToMail:
sqlFields.append( "DeferToMail" )
sqlValues.append( "0" )
if lifetime:
sqlFields.append( "Expiration" )
sqlValues.append( "TIMESTAMPADD( SECOND, %d, UTC_TIMESTAMP() )" % int( lifetime ) )
sqlInsert = "INSERT INTO `ntf_Notifications` (%s) VALUES (%s) " % ( ",".join( sqlFields ),
",".join( sqlValues ) )
result = self._update( sqlInsert )
if not result[ 'OK' ]:
return result
return S_OK( result[ 'lastRowId' ] )
def _getPilotProxyFromDIRACGroup(self, ownerDN, ownerGroup,
requiredTimeLeft):
"""
Download a limited pilot proxy with VOMS extensions depending on the group
"""
#Assign VOMS attribute
vomsAttr = CS.getVOMSAttributeForGroup(ownerGroup)
if not vomsAttr:
self.log.info(
"Downloading a proxy without VOMS extensions for %s@%s" %
(ownerDN, ownerGroup))
return gProxyManager.downloadProxy(
ownerDN,
ownerGroup,
limited=True,
requiredTimeLeft=requiredTimeLeft)
else:
self.log.info(
"Downloading a proxy with '%s' VOMS extension for %s@%s" %
(vomsAttr, ownerDN, ownerGroup))
return gProxyManager.downloadVOMSProxy(
ownerDN,
ownerGroup,
limited=True,
requiredTimeLeft=requiredTimeLeft,
requiredVOMSAttribute=vomsAttr)
def getPayloadProxyFromDIRACGroup(self,
userDN,
userGroup,
requiredTimeLeft,
token=False,
proxyToConnect=False):
"""
Download a payload proxy with VOMS extensions depending on the group
"""
#Assign VOMS attribute
vomsAttr = CS.getVOMSAttributeForGroup(userGroup)
if not vomsAttr:
gLogger.verbose(
"No voms attribute assigned to group %s when requested payload proxy"
% userGroup)
return self.downloadProxy(userDN,
userGroup,
limited=True,
requiredTimeLeft=requiredTimeLeft,
proxyToConnect=proxyToConnect,
token=token)
else:
return self.downloadVOMSProxy(userDN,
userGroup,
limited=True,
requiredTimeLeft=requiredTimeLeft,
requiredVOMSAttribute=vomsAttr,
proxyToConnect=proxyToConnect,
token=token)
def addDataset( self, name, datasetDict, numericid ):
""" Pretty print of the dataset ls output
"""
perm = datasetDict['Mode']
date = datasetDict['ModificationDate']
size = datasetDict['TotalSize']
if datasetDict.has_key('Owner'):
uname = datasetDict['Owner']
elif datasetDict.has_key('OwnerDN'):
result = CS.getUsernameForDN(datasetDict['OwnerDN'])
if result['OK']:
uname = result['Value']
else:
uname = 'unknown'
else:
uname = 'unknown'
if numericid:
uname = str( datasetDict['UID'] )
gname = 'unknown'
if datasetDict.has_key('OwnerGroup'):
gname = datasetDict['OwnerGroup']
if numericid:
gname = str( datasetDict ['GID'] )
numberOfFiles = datasetDict ['NumberOfFiles']
self.entries.append( ('s'+self.__getModeString(perm),numberOfFiles,uname,gname,size,date,name) )
def getShifterProxy(shifterType, fileName=False):
"""
This method returns a shifter's proxy
- shifterType : ProductionManager / DataManager...
"""
if fileName:
try:
os.makedirs(os.path.dirname(fileName))
except:
pass
shifterSection = "/Operations/Shifter/%s" % shifterType
userName = gConfig.getValue('%s/User' % shifterSection, '')
if not userName:
return S_ERROR("No shifter defined in %s/User" % shifterSection)
result = CS.getDNForUsername(userName)
if not result['OK']:
return result
userDN = result['Value'][0]
userGroup = gConfig.getValue('%s/Group' % shifterSection,
CS.getDefaultUserGroup())
vomsAttr = CS.getVOMSAttributeForGroup(userGroup)
if vomsAttr:
gLogger.info("Getting VOMS [%s] proxy for shifter %s@%s (%s)" %
(vomsAttr, userName, userGroup, userDN))
result = gProxyManager.downloadVOMSProxy(userDN,
userGroup,
requiredTimeLeft=4 * 43200)
else:
gLogger.info("Getting proxy for shifter %s@%s (%s)" %
(userName, userGroup, userDN))
result = gProxyManager.downloadProxy(userDN,
userGroup,
requiredTimeLeft=4 * 43200)
if not result['OK']:
return result
chain = result['Value']
result = gProxyManager.dumpProxyToFile(chain, destinationFile=fileName)
if not result['OK']:
return result
fileName = result['Value']
return S_OK({
'DN': userDN,
'username': userName,
'group': userGroup,
'chain': chain,
'proxyFile': fileName
})
def __getVOMSAttribute(self, userGroup, requiredVOMSAttribute=False):
if requiredVOMSAttribute:
return S_OK({
'attribute': requiredVOMSAttribute,
'VOMSVO': CS.getVOMSVOForGroup(userGroup)
})
csVOMSMapping = CS.getVOMSAttributeForGroup(userGroup)
if not csVOMSMapping:
return S_ERROR("No mapping defined for group %s in the CS" %
userGroup)
return S_OK({
'attribute': csVOMSMapping,
'VOMSVO': CS.getVOMSVOForGroup(userGroup)
})
def __getOwnerGroupDN(self, shifterType):
opsHelper = Operations()
userName = opsHelper.getValue(
cfgPath('BoincShifter', shifterType, 'User'), '')
if not userName:
return S_ERROR("No shifter User defined for %s" % shifterType)
result = CS.getDNForUsername(userName)
if not result['OK']:
return result
userDN = result['Value'][0]
result = CS.findDefaultGroupForDN(userDN)
if not result['OK']:
return result
defaultGroup = result['Value']
userGroup = opsHelper.getValue(
cfgPath('BoincShifter', shifterType, 'Group'), defaultGroup)
return userDN, userGroup, userName
def sendPilotAccounting(self, pilotDict):
""" Send pilot accounting record
"""
for pRef in pilotDict:
self.log.verbose('Preparing accounting record for pilot %s' % pRef)
pA = PilotAccounting()
pA.setEndTime(pilotDict[pRef]['LastUpdateTime'])
pA.setStartTime(pilotDict[pRef]['SubmissionTime'])
retVal = CS.getUsernameForDN(pilotDict[pRef]['OwnerDN'])
if not retVal['OK']:
userName = '******'
self.log.error("Can't determine username for dn:",
pilotDict[pRef]['OwnerDN'])
else:
userName = retVal['Value']
pA.setValueByKey('User', userName)
pA.setValueByKey('UserGroup', pilotDict[pRef]['OwnerGroup'])
result = getSiteForCE(pilotDict[pRef]['DestinationSite'])
if result['OK'] and result['Value'].strip():
pA.setValueByKey('Site', result['Value'].strip())
else:
pA.setValueByKey('Site', 'Unknown')
pA.setValueByKey('GridCE', pilotDict[pRef]['DestinationSite'])
pA.setValueByKey('GridMiddleware', pilotDict[pRef]['GridType'])
pA.setValueByKey('GridResourceBroker', pilotDict[pRef]['Broker'])
pA.setValueByKey('GridStatus', pilotDict[pRef]['Status'])
if not 'Jobs' in pilotDict[pRef]:
pA.setValueByKey('Jobs', 0)
else:
pA.setValueByKey('Jobs', len(pilotDict[pRef]['Jobs']))
self.log.info("Adding accounting record for pilot %s" %
pilotDict[pRef]['PilotID'])
retVal = gDataStoreClient.addRegister(pA)
if not retVal['OK']:
self.log.error('Failed to send accounting info for pilot ',
pRef)
else:
# Set up AccountingSent flag
result = pilotAgentsDB.setAccountingFlag(pRef)
if not result['OK']:
self.log.error('Failed to set accounting flag for pilot ',
pRef)
self.log.info('Committing accounting records for %d pilots' %
len(pilotDict))
result = gDataStoreClient.commit()
if result['OK']:
for pRef in pilotDict:
self.log.verbose('Setting AccountingSent flag for pilot %s' %
pRef)
result = pilotAgentsDB.setAccountingFlag(pRef)
if not result['OK']:
self.log.error('Failed to set accounting flag for pilot ',
pRef)
else:
return result
return S_OK()
def __setupManagerProxies( self ):
""" setup grid proxy for all defined managers """
oHelper = Operations()
shifters = oHelper.getSections( "Shifter" )
if not shifters["OK"]:
self.log.error( shifters["Message"] )
return shifters
shifters = shifters["Value"]
for shifter in shifters:
shifterDict = oHelper.getOptionsDict( "Shifter/%s" % shifter )
if not shifterDict["OK"]:
self.log.error( shifterDict["Message"] )
continue
userName = shifterDict["Value"].get( "User", "" )
userGroup = shifterDict["Value"].get( "Group", "" )
userDN = CS.getDNForUsername( userName )
if not userDN["OK"]:
self.log.error( userDN["Message"] )
continue
userDN = userDN["Value"][0]
vomsAttr = CS.getVOMSAttributeForGroup( userGroup )
if vomsAttr:
self.log.debug( "getting VOMS [%s] proxy for shifter %s@%s (%s)" % ( vomsAttr, userName,
userGroup, userDN ) )
getProxy = gProxyManager.downloadVOMSProxyToFile( userDN, userGroup,
requiredTimeLeft = 1200,
cacheTime = 4 * 43200 )
else:
self.log.debug( "getting proxy for shifter %s@%s (%s)" % ( userName, userGroup, userDN ) )
getProxy = gProxyManager.downloadProxyToFile( userDN, userGroup,
requiredTimeLeft = 1200,
cacheTime = 4 * 43200 )
if not getProxy["OK"]:
self.log.error( getProxy["Message" ] )
return S_ERROR( "unable to setup shifter proxy for %s: %s" % ( shifter, getProxy["Message"] ) )
chain = getProxy["chain"]
fileName = getProxy["Value" ]
self.log.debug( "got %s: %s %s" % ( shifter, userName, userGroup ) )
self.__managersDict[shifter] = { "ShifterDN" : userDN,
"ShifterName" : userName,
"ShifterGroup" : userGroup,
"Chain" : chain,
"ProxyFile" : fileName }
return S_OK()
def _getBaseStub( self ):
newKwargs = dict( self.kwargs )
#Set DN
if 'DN' in self.__idDict and not self.KW_DELEGATED_DN in newKwargs:
newKwargs[ self.KW_DELEGATED_DN ] = self.__idDict[ 'DN' ]
#Discover group
if not self.KW_DELEGATED_GROUP in newKwargs:
if 'group' in self.__idDict:
newKwargs[ self.KW_DELEGATED_GROUP ] = self.__idDict[ 'group' ]
else:
if self.KW_DELEGATED_DN in newKwargs:
if CS.getUsernameForDN( newKwargs[ self.KW_DELEGATED_DN ] )[ 'OK' ]:
newKwargs[ self.KW_DELEGATED_GROUP ] = CS.getDefaultUserGroup()
if CS.getHostnameForDN( newKwargs[ self.KW_DELEGATED_DN ] )[ 'OK' ]:
newKwargs[ self.KW_DELEGATED_GROUP ] = self.VAL_EXTRA_CREDENTIALS_HOST
if 'useCertificates' in newKwargs:
del( newKwargs[ 'useCertificates' ] )
return ( self._destinationSrv, newKwargs )
def addFile(self, name, fileDict, repDict, numericid):
""" Pretty print of the file ls output
"""
perm = fileDict['Mode']
date = fileDict['ModificationDate']
#nlinks = fileDict.get('NumberOfLinks',0)
nreplicas = len(repDict)
size = fileDict['Size']
if fileDict.has_key('Owner'):
uname = fileDict['Owner']
elif fileDict.has_key('OwnerDN'):
result = CS.getUsernameForDN(fileDict['OwnerDN'])
if result['OK']:
uname = result['Value']
else:
uname = 'unknown'
else:
uname = 'unknown'
if numericid:
uname = str(fileDict['UID'])
if fileDict.has_key('OwnerGroup'):
gname = fileDict['OwnerGroup']
elif fileDict.has_key('OwnerRole'):
groups = CS.getGroupsWithVOMSAttribute('/' + fileDict['OwnerRole'])
if groups:
if len(groups) > 1:
gname = groups[0]
default_group = gConfig.getValue('/Registry/DefaultGroup',
'unknown')
if default_group in groups:
gname = default_group
else:
gname = groups[0]
else:
gname = 'unknown'
else:
gname = 'unknown'
if numericid:
gname = str(fileDict['GID'])
self.entries.append(('-' + self.__getModeString(perm), nreplicas,
uname, gname, size, date, name))
def export_getContents(self, selDict, sortDict, start, limit):
"""
Retrieve the contents of the DB
"""
credDict = self.getRemoteCredentials()
if not Properties.PROXY_MANAGEMENT in credDict['properties']:
result = CS.getDNForUsername(credDict['username'])
if not result['OK']:
return S_ERROR("You are not a valid user!")
selDict['UserDN'] = result['Value']
return gProxyDB.getProxiesContent(selDict, sortDict, start, limit)
def export_getContents( self, selDict, sortDict, start, limit ):
"""
Retrieve the contents of the DB
"""
credDict = self.getRemoteCredentials()
if not Properties.PROXY_MANAGEMENT in credDict[ 'properties' ]:
result = CS.getDNForUsername( credDict[ 'username' ] )
if not result[ 'OK' ]:
return S_ERROR( "You are not a valid user!" )
selDict[ 'UserDN' ] = result[ 'Value' ]
return gProxyDB.getProxiesContent( selDict, sortDict, start, limit )
def getAssigneeGroupsForUser( self, user ):
if user not in CS.getAllUsers():
return S_ERROR( "%s is an unknown user" % user )
result = self._escapeString( user )
if not result[ 'OK' ]:
return result
user = result[ 'Value' ]
result = self._query( "SELECT AssigneeGroup from `ntf_AssigneeGroups` WHERE User=%s" % user )
if not result[ 'OK' ]:
return result
return S_OK( [ row[0] for row in result[ 'Value' ] ] )
def getHostNickName( self, credDict ):
"""
Discover the host nickname associated to the DN.
The nickname will be included in the credentials dictionary.
:type credDict: dictionary
:param credDict: Credentials to ckeck
:return: Boolean specifying whether the nickname was found
"""
if self.KW_DN not in credDict:
return True
if self.KW_GROUP not in credDict:
return False
retVal = CS.getHostnameForDN( credDict[ self.KW_DN ] )
if not retVal[ 'OK' ]:
gLogger.warn( "Cannot find hostname for DN %s: %s" % ( credDict[ self.KW_DN ], retVal[ 'Message' ] ) )
return False
credDict[ self.KW_USERNAME ] = retVal[ 'Value' ]
credDict[ self.KW_PROPERTIES ] = CS.getPropertiesForHost( credDict[ self.KW_USERNAME ], [] )
return True
def getShifterProxy(shifterType, fileName=False):
"""
This method returns a shifter's proxy
- shifterType : ProductionManager / DataManager...
"""
if fileName:
try:
os.makedirs(os.path.dirname(fileName))
except OSError:
pass
opsHelper = Operations()
userName = opsHelper.getValue(cfgPath("Shifter", shifterType, "User"), "")
if not userName:
return S_ERROR("No shifter User defined for %s" % shifterType)
result = CS.getDNForUsername(userName)
if not result["OK"]:
return result
userDN = result["Value"][0]
result = CS.findDefaultGroupForDN(userDN)
if not result["OK"]:
return result
defaultGroup = result["Value"]
userGroup = opsHelper.getValue(cfgPath("Shifter", shifterType, "Group"), defaultGroup)
vomsAttr = CS.getVOMSAttributeForGroup(userGroup)
if vomsAttr:
gLogger.info("Getting VOMS [%s] proxy for shifter %s@%s (%s)" % (vomsAttr, userName, userGroup, userDN))
result = gProxyManager.downloadVOMSProxyToFile(
userDN, userGroup, filePath=fileName, requiredTimeLeft=1200, cacheTime=4 * 43200
)
else:
gLogger.info("Getting proxy for shifter %s@%s (%s)" % (userName, userGroup, userDN))
result = gProxyManager.downloadProxyToFile(
userDN, userGroup, filePath=fileName, requiredTimeLeft=1200, cacheTime=4 * 43200
)
if not result["OK"]:
return result
chain = result["chain"]
fileName = result["Value"]
return S_OK({"DN": userDN, "username": userName, "group": userGroup, "chain": chain, "proxyFile": fileName})