def export_getPilotStatistics(attribute, selectDict):
""" Get pilot statistics distribution per attribute value with a given selection
"""
startDate = selectDict.get('FromDate', None)
if startDate:
del selectDict['FromDate']
if startDate is None:
startDate = selectDict.get('LastUpdate', None)
if startDate:
del selectDict['LastUpdate']
endDate = selectDict.get('ToDate', None)
if endDate:
del selectDict['ToDate']
result = pilotDB.getCounters('PilotAgents', [attribute], selectDict,
newer=startDate,
older=endDate,
timeStamp='LastUpdateTime')
statistics = {}
if result['OK']:
for status, count in result['Value']:
if "OwnerDN" in status:
userName = getUsernameForDN(status['OwnerDN'])
if userName['OK']:
status['OwnerDN'] = userName['Value']
statistics[status['OwnerDN']] = count
else:
statistics[status[attribute]] = count
return S_OK(statistics)
def getUserNameAndGroup(info):
""" Get the user name and group from the DN and VOMS role
"""
global dnCache, roleCache
owner = {}
username = dnCache.get(info['OwnerDN'])
if not username:
result = getUsernameForDN(info['OwnerDN'])
if result['OK']:
username = result['Value']
dnCache[info['OwnerDN']] = username
elif "No username" in result['Message']:
username = '******'
dnCache[info['OwnerDN']] = username
if username and username != 'Unknown':
groups = roleCache.get('/'+info['OwnerRole'])
if not groups:
groups = getGroupsWithVOMSAttribute('/'+info['OwnerRole'])
roleCache['/'+info['OwnerRole']] = groups
if groups:
owner['username'] = username
owner['group'] = groups[0]
return owner
def getPilotMonitorSelectors(self):
""" Get distinct values for the Pilot Monitor page selectors
"""
paramNames = [
'OwnerDN', 'OwnerGroup', 'GridType', 'Broker', 'Status',
'DestinationSite', 'GridSite'
]
resultDict = {}
for param in paramNames:
result = self.getDistinctAttributeValues('PilotAgents', param)
if result['OK']:
resultDict[param] = result['Value']
else:
resultDict = []
if param == "OwnerDN":
userList = []
for dn in result['Value']:
resultUser = getUsernameForDN(dn)
if resultUser['OK']:
userList.append(resultUser['Value'])
resultDict["Owner"] = userList
return S_OK(resultDict)
def getUserRightsForJob(self, jobID):
""" Get access rights to job with jobID for the user specified by
userDN/userGroup
"""
result = self.jobDB.getJobAttributes(jobID, ['OwnerDN', 'OwnerGroup'])
if not result['OK']:
return result
elif result['Value']:
owner = result['Value']['OwnerDN']
group = result['Value']['OwnerGroup']
result = getUsernameForDN(owner)
ownerName = ''
if result['OK']:
ownerName = result['Value']
result = self.getJobPolicy(owner, group)
if self.userName and self.userName == ownerName and self.userGroup == group:
result['UserIsOwner'] = True
else:
result['UserIsOwner'] = False
return result
else:
return S_ERROR('Job not found')
def getPilotMonitorSelectors(self):
"""Get distinct values for the Pilot Monitor page selectors"""
paramNames = [
"OwnerDN", "OwnerGroup", "GridType", "Broker", "Status",
"DestinationSite", "GridSite"
]
resultDict = {}
for param in paramNames:
result = self.getDistinctAttributeValues("PilotAgents", param)
if result["OK"]:
resultDict[param] = result["Value"]
else:
resultDict = []
if param == "OwnerDN":
userList = []
for dn in result["Value"]:
resultUser = getUsernameForDN(dn)
if resultUser["OK"]:
userList.append(resultUser["Value"])
resultDict["Owner"] = userList
return S_OK(resultDict)
def __sendAccounting( ftsJob, ownerDN ):
""" prepare and send DataOperation to AccouringDB """
dataOp = DataOperation()
dataOp.setStartTime( fromString( ftsJob.SubmitTime ) )
dataOp.setEndTime( fromString( ftsJob.LastUpdate ) )
accountingDict = dict()
accountingDict["OperationType"] = "ReplicateAndRegister"
username = getUsernameForDN( ownerDN )
if not username["OK"]:
username = ownerDN
else:
username = username["Value"]
accountingDict["User"] = username
accountingDict["Protocol"] = "FTS"
# accountingDict['RegistrationTime'] = 0
# accountingDict['RegistrationOK'] = 0
# accountingDict['RegistrationTotal'] = 0
accountingDict["TransferOK"] = len( [ f for f in ftsJob if f.Status == "Finished" ] )
accountingDict["TransferTotal"] = len( ftsJob )
accountingDict["TransferSize"] = ftsJob.Size
accountingDict["FinalStatus"] = ftsJob.Status
accountingDict["Source"] = ftsJob.SourceSE
accountingDict["Destination"] = ftsJob.TargetSE
dt = ftsJob.LastUpdate - ftsJob.SubmitTime
transferTime = dt.days * 86400 + dt.seconds
accountingDict["TransferTime"] = transferTime
dataOp.setValuesFromDict( accountingDict )
dataOp.commit()
def _addOperationForTransformations(operationsOnTransformationDict,
operation,
transformations,
owner=None,
ownerGroup=None,
ownerDN=None):
"""Fill the operationsOnTransformationDict"""
transformationIDsAndBodies = [
(transformation['TransformationID'], transformation['Body'],
transformation['AuthorDN'], transformation['AuthorGroup'])
for transformation in transformations['Value']
]
for transID, body, t_ownerDN, t_ownerGroup in transformationIDsAndBodies:
if transID in operationsOnTransformationDict:
operationsOnTransformationDict[transID]['Operations'].append(
operation)
else:
operationsOnTransformationDict[transID] = {
'Body':
body,
'Operations': [operation],
'Owner':
owner if owner else getUsernameForDN(t_ownerDN)['Value'],
'OwnerGroup':
ownerGroup if owner else t_ownerGroup,
'OwnerDN':
ownerDN if owner else t_ownerDN
}
def _addOperationForTransformations(
operationsOnTransformationDict,
operation,
transformations,
owner=None,
ownerGroup=None,
ownerDN=None,
):
"""Fill the operationsOnTransformationDict"""
transformationIDsAndBodies = (
(
transformation["TransformationID"],
transformation["Body"],
transformation["AuthorDN"],
transformation["AuthorGroup"],
)
for transformation in transformations["Value"]
)
for transID, body, t_ownerDN, t_ownerGroup in transformationIDsAndBodies:
if transID in operationsOnTransformationDict:
operationsOnTransformationDict[transID]["Operations"].append(operation)
else:
operationsOnTransformationDict[transID] = {
"TransformationID": transID,
"Body": body,
"Operations": [operation],
"Owner": owner if owner else getUsernameForDN(t_ownerDN)["Value"],
"OwnerGroup": ownerGroup if owner else t_ownerGroup,
"OwnerDN": ownerDN if owner else t_ownerDN,
}
def getUserNameAndGroup(info):
""" Get the user name and group from the DN and VOMS role
"""
global dnCache, roleCache
owner = {}
if "OwnerDN" not in info:
return owner
username = dnCache.get(info.get('OwnerDN'))
if not username:
result = getUsernameForDN(info.get('OwnerDN', 'Unknown'))
if result['OK']:
username = result['Value']
dnCache[info['OwnerDN']] = username
elif "No username" in result['Message']:
username = '******'
dnCache[info['OwnerDN']] = username
if username and username != 'Unknown':
groups = roleCache.get('/' + info.get('OwnerRole'))
if not groups:
groups = getGroupsWithVOMSAttribute('/' + info['OwnerRole'])
roleCache['/' + info['OwnerRole']] = groups
if groups:
owner['username'] = username
owner['group'] = groups[0]
return owner
def export_getPilotStatistics ( attribute, selectDict ):
""" Get pilot statistics distribution per attribute value with a given selection
"""
startDate = selectDict.get( 'FromDate', None )
if startDate:
del selectDict['FromDate']
if startDate is None:
startDate = selectDict.get( 'LastUpdate', None )
if startDate:
del selectDict['LastUpdate']
endDate = selectDict.get( 'ToDate', None )
if endDate:
del selectDict['ToDate']
result = pilotDB.getCounters( 'PilotAgents', [attribute], selectDict,
newer = startDate,
older = endDate,
timeStamp = 'LastUpdateTime' )
statistics = {}
if result['OK']:
for status, count in result['Value']:
if "OwnerDN" in status:
userName = getUsernameForDN( status['OwnerDN'] )
if userName['OK']:
status['OwnerDN'] = userName['Value']
statistics[status[selector]] = count
return S_OK( statistics )
def getUserNameAndGroup(info):
"""Get the user name and group from the DN and VOMS role"""
global dnCache, roleCache
owner = {}
if "OwnerDN" not in info:
return owner
username = dnCache.get(info.get("OwnerDN"))
if not username:
result = getUsernameForDN(info.get("OwnerDN", "Unknown"))
if result["OK"]:
username = result["Value"]
dnCache[info["OwnerDN"]] = username
elif "No username" in result["Message"]:
username = "******"
dnCache[info["OwnerDN"]] = username
if username and username != "Unknown":
groups = roleCache.get("/" + info.get("OwnerRole"))
if not groups:
groups = getGroupsWithVOMSAttribute("/" + info["OwnerRole"])
roleCache["/" + info["OwnerRole"]] = groups
if groups:
owner["username"] = username
owner["group"] = groups[0]
return owner
def getUserRightsForJob( self, jobID ):
""" Get access rights to job with jobID for the user specified by
userDN/userGroup
"""
result = self.jobDB.getJobAttributes( jobID, [ 'OwnerDN', 'OwnerGroup' ] )
if not result['OK']:
return result
elif result['Value']:
owner = result['Value']['OwnerDN']
group = result['Value']['OwnerGroup']
result = getUsernameForDN(owner)
ownerName = ''
if result['OK']:
ownerName = result['Value']
result = self.getJobPolicy( owner, group )
if self.userName and self.userName == ownerName and self.userGroup == group:
result[ 'UserIsOwner' ] = True
else:
result[ 'UserIsOwner' ] = False
return result
else:
return S_ERROR('Job not found')
def __init__(self, userDN, userGroup, userProperties):
self.userDN = userDN
self.userName = ''
result = getUsernameForDN(userDN)
if result['OK']:
self.userName = result['Value']
self.userGroup = userGroup
self.userProperties = userProperties
self.jobDB = None
def __init__( self, userDN, userGroup, userProperties ):
self.userDN = userDN
self.userName = ''
result = getUsernameForDN(userDN)
if result['OK']:
self.userName = result['Value']
self.userGroup = userGroup
self.userProperties = userProperties
self.jobDB = None
def authenticate_user(self, credential):
"""Authorize user
:param dict credential: credential (token payload)
:return: str or bool
"""
result = getUsernameForDN(wrapIDAsDN(credential["sub"]))
if not result["OK"]:
self.server.log.error(result["Message"])
return result.get("Value")
def __init__(self, userDN, userGroup, allInfo=True):
self.userDN = userDN
self.userName = ''
result = getUsernameForDN(userDN)
if result['OK']:
self.userName = result['Value']
self.userGroup = userGroup
self.userProperties = getPropertiesForGroup(userGroup, [])
self.jobDB = None
self.allInfo = allInfo
self.__permissions = {}
self.__getUserJobPolicy()
def __init__( self, userDN, userGroup, allInfo=True ):
self.userDN = userDN
self.userName = ''
result = getUsernameForDN(userDN)
if result['OK']:
self.userName = result['Value']
self.userGroup = userGroup
self.userProperties = getPropertiesForGroup( userGroup, [] )
self.jobDB = None
self.allInfo = allInfo
self.__permissions = {}
self.__getUserJobPolicy()
def export_banSite(self, site, comment='No comment'):
""" Ban the given site in the site mask
"""
result = self.getRemoteCredentials()
dn = result['DN']
result = getUsernameForDN(dn)
if result['OK']:
author = result['Value']
else:
author = dn
result = jobDB.banSiteInMask(site, author, comment)
return result
def export_banSite(self, site,comment='No comment'):
""" Ban the given site in the site mask
"""
result = self.getRemoteCredentials()
dn = result['DN']
result = getUsernameForDN(dn)
if result['OK']:
author = result['Value']
else:
author = dn
result = jobDB.banSiteInMask(site,author,comment)
return result
def parseIdPAuthorizationResponse(self, response, session):
"""Fill session by user profile, tokens, comment, OIDC authorize status, etc.
Prepare dict with user parameters, if DN is absent there try to get it.
Create new or modify existing DIRAC user and store the session
:param dict response: authorization response
:param str session: session
:return: S_OK(dict)/S_ERROR()
"""
providerName = session.pop("Provider")
sLog.debug("Try to parse authentification response from %s:\n" % providerName, pprint.pformat(response))
# Parse response
result = self.idps.getIdProvider(providerName)
if not result["OK"]:
return result
idpObj = result["Value"]
result = idpObj.parseAuthResponse(response, session)
if not result["OK"]:
return result
# FINISHING with IdP
# As a result of authentication we will receive user credential dictionary
credDict, payload = result["Value"]
sLog.debug("Read profile:", pprint.pformat(credDict))
# Is ID registred?
result = getUsernameForDN(credDict["DN"])
if not result["OK"]:
comment = f"ID {credDict['ID']} is not registred in DIRAC. "
payload.update(idpObj.getUserProfile().get("Value", {}))
result = self.__registerNewUser(providerName, payload)
if result["OK"]:
comment += "Administrators have been notified about you."
else:
comment += "Please, contact the DIRAC administrators."
# Notify user about problem
html = getHTML("unregistered user!", info=comment, theme="warning")
return S_ERROR(html)
credDict["username"] = result["Value"]
# Update token for user. This token will be stored separately in the database and
# updated from time to time. This token will never be transmitted,
# it will be used to make exchange token requests.
result = self.tokenCli.updateToken(idpObj.token, credDict["ID"], idpObj.name)
return S_OK(credDict) if result["OK"] else result
def _addOperationForTransformations(operationsOnTransformationDict, operation, transformations,
owner=None, ownerGroup=None, ownerDN=None):
"""Fill the operationsOnTransformationDict"""
transformationIDsAndBodies = [(transformation['TransformationID'],
transformation['Body'],
transformation['AuthorDN'],
transformation['AuthorGroup']) for transformation in transformations['Value']]
for transID, body, t_ownerDN, t_ownerGroup in transformationIDsAndBodies:
if transID in operationsOnTransformationDict:
operationsOnTransformationDict[transID]['Operations'].append(operation)
else:
operationsOnTransformationDict[transID] = {'Body': body, 'Operations': [operation],
'Owner': owner if owner else getUsernameForDN(t_ownerDN)['Value'],
'OwnerGroup': ownerGroup if owner else t_ownerGroup,
'OwnerDN': ownerDN if owner else t_ownerDN}
def export_getPilotStatistics(cls, attribute, selectDict):
"""Get pilot statistics distribution per attribute value with a given selection"""
startDate = selectDict.get("FromDate", None)
if startDate:
del selectDict["FromDate"]
if startDate is None:
startDate = selectDict.get("LastUpdate", None)
if startDate:
del selectDict["LastUpdate"]
endDate = selectDict.get("ToDate", None)
if endDate:
del selectDict["ToDate"]
# Owner attribute is not part of PilotAgentsDB
# It has to be converted into a OwnerDN
owners = selectDict.get("Owner")
if owners:
ownerDNs = []
for owner in owners:
result = getDNForUsername(owner)
if not result["OK"]:
return result
ownerDNs.append(result["Value"])
selectDict["OwnerDN"] = ownerDNs
del selectDict["Owner"]
result = cls.pilotAgentsDB.getCounters("PilotAgents", [attribute],
selectDict,
newer=startDate,
older=endDate,
timeStamp="LastUpdateTime")
statistics = {}
if result["OK"]:
for status, count in result["Value"]:
if "OwnerDN" in status:
userName = getUsernameForDN(status["OwnerDN"])
if userName["OK"]:
status["OwnerDN"] = userName["Value"]
statistics[status["OwnerDN"]] = count
else:
statistics[status[attribute]] = count
return S_OK(statistics)
def getJobPolicy(self, jobOwnerDN='', jobOwnerGroup=''):
""" Get the job operations rights for a job owned by jobOwnerDN/jobOwnerGroup
for a user with userDN/userGroup.
Returns a dictionary of various operations rights
"""
# Can not do anything by default
permDict = {}
for r in ALL_RIGHTS:
permDict[r] = False
# Anybody can get info about the jobs
permDict[RIGHT_GET_INFO] = True
# Give JobAdmin permission if needed
if Properties.JOB_ADMINISTRATOR in self.userProperties:
for r in PROPERTY_RIGHTS[Properties.JOB_ADMINISTRATOR]:
permDict[r] = True
# Give JobAdmin permission if needed
if Properties.NORMAL_USER in self.userProperties:
for r in PROPERTY_RIGHTS[Properties.NORMAL_USER]:
permDict[r] = True
# Give permissions of the generic pilot
if Properties.GENERIC_PILOT in self.userProperties:
for r in PROPERTY_RIGHTS[Properties.GENERIC_PILOT]:
permDict[r] = True
# Job Owner can do everything with his jobs
result = getUsernameForDN(jobOwnerDN)
jobOwnerName = ''
if result['OK']:
jobOwnerName = result['Value']
if jobOwnerName and self.userName and jobOwnerName == self.userName:
for r in OWNER_RIGHTS:
permDict[r] = True
# Members of the same group sharing their jobs can do everything
if jobOwnerGroup == self.userGroup:
if Properties.JOB_SHARING in self.userProperties:
for right in GROUP_RIGHTS:
permDict[right] = True
return S_OK(permDict)
def getJobPolicy( self, jobOwnerDN = '', jobOwnerGroup = '' ):
""" Get the job operations rights for a job owned by jobOwnerDN/jobOwnerGroup
for a user with userDN/userGroup.
Returns a dictionary of various operations rights
"""
# Can not do anything by default
permDict = {}
for r in ALL_RIGHTS:
permDict[r] = False
# Anybody can get info about the jobs
permDict[ RIGHT_GET_INFO ] = True
# Give JobAdmin permission if needed
if Properties.JOB_ADMINISTRATOR in self.userProperties:
for r in PROPERTY_RIGHTS[ Properties.JOB_ADMINISTRATOR ]:
permDict[ r ] = True
# Give JobAdmin permission if needed
if Properties.NORMAL_USER in self.userProperties:
for r in PROPERTY_RIGHTS[ Properties.NORMAL_USER ]:
permDict[ r ] = True
# Give permissions of the generic pilot
if Properties.GENERIC_PILOT in self.userProperties:
for r in PROPERTY_RIGHTS[ Properties.GENERIC_PILOT ]:
permDict[ r ] = True
# Job Owner can do everything with his jobs
result = getUsernameForDN(jobOwnerDN)
jobOwnerName = ''
if result['OK']:
jobOwnerName = result['Value']
if jobOwnerName and self.userName and jobOwnerName == self.userName:
for r in OWNER_RIGHTS:
permDict[r] = True
# Members of the same group sharing their jobs can do everything
if jobOwnerGroup == self.userGroup:
if Properties.JOB_SHARING in self.userProperties:
for right in GROUP_RIGHTS:
permDict[right] = True
return S_OK( permDict )
def __sendAccounting(ftsJob, ownerDN):
""" prepare and send DataOperation to AccouringDB """
dataOp = DataOperation()
dataOp.setStartTime(fromString(ftsJob.SubmitTime))
dataOp.setEndTime(fromString(ftsJob.LastUpdate))
accountingDict = dict()
accountingDict["OperationType"] = "ReplicateAndRegister"
username = getUsernameForDN(ownerDN)
if not username["OK"]:
username = ownerDN
else:
username = username["Value"]
accountingDict["User"] = username
accountingDict[
"Protocol"] = "FTS3" if 'fts3' in ftsJob.FTSServer.lower(
) else 'FTS'
accountingDict['ExecutionSite'] = ftsJob.FTSServer
accountingDict['RegistrationTime'] = ftsJob._regTime
accountingDict['RegistrationOK'] = ftsJob._regSuccess
accountingDict['RegistrationTotal'] = ftsJob._regTotal
accountingDict["TransferOK"] = len(
[f for f in ftsJob if f.Status in FTSFile.SUCCESS_STATES])
accountingDict["TransferTotal"] = len(ftsJob)
accountingDict["TransferSize"] = ftsJob.Size - ftsJob.FailedSize
accountingDict["FinalStatus"] = ftsJob.Status
accountingDict["Source"] = ftsJob.SourceSE
accountingDict["Destination"] = ftsJob.TargetSE
dt = ftsJob.LastUpdate - ftsJob.SubmitTime
transferTime = dt.days * 86400 + dt.seconds
accountingDict["TransferTime"] = transferTime
# accountingDict['TransferTime'] = sum( [f._duration for f in ftsJob])
dataOp.setValuesFromDict(accountingDict)
dataOp.commit()
def setUp(self):
self.host = "localhost"
self.notificationPort = 9154
self.rootPwd = ""
self.csClient = CSAPI()
self.monitoringClient = ComponentMonitoringClient()
self.client = SystemAdministratorClientCLI(self.host)
self.csClient.downloadCSData()
result = self.csClient.getCurrentCFG()
if not result["OK"]:
raise Exception(result["Message"])
cfg = result["Value"]
setup = cfg.getOption("DIRAC/Setup", "dirac-JenkinsSetup")
self.frameworkSetup = cfg.getOption("DIRAC/Setups/" + setup +
"/Framework")
self.rootPwd = cfg.getOption("Systems/Databases/Password")
self.diracPwd = self.rootPwd
result = getProxyInfo()
if not result["OK"]:
raise Exception(result["Message"])
chain = result["Value"]["chain"]
result = chain.getCertInChain(-1)
if not result["OK"]:
raise Exception(result["Message"])
result = result["Value"].getSubjectDN()
if not result["OK"]:
raise Exception(result["Message"])
userDN = result["Value"]
result = getUsernameForDN(userDN)
if not result["OK"]:
raise Exception(result["Message"])
self.user = result["Value"]
if not self.user:
self.user = "******"
def setUp(self):
self.host = 'localhost'
self.notificationPort = 9154
self.rootPwd = ''
self.csClient = CSAPI()
self.monitoringClient = ComponentMonitoringClient()
self.client = SystemAdministratorClientCLI(self.host)
self.csClient.downloadCSData()
result = self.csClient.getCurrentCFG()
if not result['OK']:
raise Exception(result['Message'])
cfg = result['Value']
setup = cfg.getOption('DIRAC/Setup', 'dirac-JenkinsSetup')
self.frameworkSetup = cfg.getOption('DIRAC/Setups/' + setup +
'/Framework')
self.rootPwd = cfg.getOption('Systems/Databases/Password')
self.diracPwd = self.rootPwd
result = getProxyInfo()
if not result['OK']:
raise Exception(result['Message'])
chain = result['Value']['chain']
result = chain.getCertInChain(-1)
if not result['OK']:
raise Exception(result['Message'])
result = result['Value'].getSubjectDN()
if not result['OK']:
raise Exception(result['Message'])
userDN = result['Value']
result = getUsernameForDN(userDN)
if not result['OK']:
raise Exception(result['Message'])
self.user = result['Value']
if not self.user:
self.user = '******'
def setUp(self):
self.host = 'localhost'
self.notificationPort = 9154
self.rootPwd = ''
self.csClient = CSAPI()
self.monitoringClient = ComponentMonitoringClient()
self.client = SystemAdministratorClientCLI(self.host)
self.csClient.downloadCSData()
result = self.csClient.getCurrentCFG()
if not result['OK']:
raise Exception(result['Message'])
cfg = result['Value']
setup = cfg.getOption('DIRAC/Setup', 'dirac-JenkinsSetup')
self.frameworkSetup = cfg.getOption('DIRAC/Setups/' + setup + '/Framework')
self.rootPwd = cfg.getOption('Systems/Databases/Password')
self.diracPwd = self.rootPwd
result = getProxyInfo()
if not result['OK']:
raise Exception(result['Message'])
chain = result['Value']['chain']
result = chain.getCertInChain(-1)
if not result['OK']:
raise Exception(result['Message'])
result = result['Value'].getSubjectDN()
if not result['OK']:
raise Exception(result['Message'])
userDN = result['Value']
result = getUsernameForDN(userDN)
if not result['OK']:
raise Exception(result['Message'])
self.user = result['Value']
if not self.user:
self.user = '******'
def __sendAccounting( ftsJob, ownerDN ):
""" prepare and send DataOperation to AccouringDB """
dataOp = DataOperation()
dataOp.setStartTime( fromString( ftsJob.SubmitTime ) )
dataOp.setEndTime( fromString( ftsJob.LastUpdate ) )
accountingDict = dict()
accountingDict["OperationType"] = "ReplicateAndRegister"
username = getUsernameForDN( ownerDN )
if not username["OK"]:
username = ownerDN
else:
username = username["Value"]
accountingDict["User"] = username
accountingDict["Protocol"] = "FTS3" if 'fts3' in ftsJob.FTSServer.lower() else 'FTS'
accountingDict['ExecutionSite'] = ftsJob.FTSServer
accountingDict['RegistrationTime'] = ftsJob._regTime
accountingDict['RegistrationOK'] = ftsJob._regSuccess
accountingDict['RegistrationTotal'] = ftsJob._regTotal
accountingDict["TransferOK"] = len( [ f for f in ftsJob if f.Status in FTSFile.SUCCESS_STATES ] )
accountingDict["TransferTotal"] = len( ftsJob )
accountingDict["TransferSize"] = ftsJob.Size - ftsJob.FailedSize
accountingDict["FinalStatus"] = ftsJob.Status
accountingDict["Source"] = ftsJob.SourceSE
accountingDict["Destination"] = ftsJob.TargetSE
# dt = ftsJob.LastUpdate - ftsJob.SubmitTime
# transferTime = dt.days * 86400 + dt.seconds
# accountingDict["TransferTime"] = transferTime
accountingDict['TransferTime'] = sum( [int( f._duration ) for f in ftsJob if f.Status in FTSFile.SUCCESS_STATES ] )
dataOp.setValuesFromDict( accountingDict )
dataOp.commit()
def getPilotMonitorSelectors( self ):
""" Get distinct values for the Pilot Monitor page selectors
"""
paramNames = ['OwnerDN', 'OwnerGroup', 'GridType', 'Broker',
'Status', 'DestinationSite', 'GridSite']
resultDict = {}
for param in paramNames:
result = self.getDistinctAttributeValues( 'PilotAgents', param )
if result['OK']:
resultDict[param] = result['Value']
else:
resultDict = []
if param == "OwnerDN":
userList = []
for dn in result['Value']:
resultUser = getUsernameForDN( dn )
if resultUser['OK']:
userList.append( resultUser['Value'] )
resultDict["Owner"] = userList
return S_OK( resultDict )
def __sendAccounting(ftsJob, ownerDN):
""" prepare and send DataOperation to AccouringDB """
dataOp = DataOperation()
dataOp.setStartTime(fromString(ftsJob.SubmitTime))
dataOp.setEndTime(fromString(ftsJob.LastUpdate))
accountingDict = dict()
accountingDict["OperationType"] = "ReplicateAndRegister"
username = getUsernameForDN(ownerDN)
if not username["OK"]:
username = ownerDN
else:
username = username["Value"]
accountingDict["User"] = username
accountingDict["Protocol"] = "FTS"
# accountingDict['RegistrationTime'] = 0
# accountingDict['RegistrationOK'] = 0
# accountingDict['RegistrationTotal'] = 0
accountingDict["TransferOK"] = len(
[f for f in ftsJob if f.Status == "Finished"])
accountingDict["TransferTotal"] = len(ftsJob)
accountingDict["TransferSize"] = ftsJob.Size
accountingDict["FinalStatus"] = ftsJob.Status
accountingDict["Source"] = ftsJob.SourceSE
accountingDict["Destination"] = ftsJob.TargetSE
dt = ftsJob.LastUpdate - ftsJob.SubmitTime
transferTime = dt.days * 86400 + dt.seconds
accountingDict["TransferTime"] = transferTime
dataOp.setValuesFromDict(accountingDict)
dataOp.commit()
def web_getStatisticsData( self ):
req = self.__request()
paletteColor = Palette()
RPC = RPCClient( "WorkloadManagement/WMSAdministrator" )
selector = self.request.arguments["statsField"][0]
if selector == 'Site':
selector = "GridSite"
if selector == "Computing Element":
selector = "DestinationSite"
elif selector == "Owner Group":
selector = "OwnerGroup"
elif selector == "Owner":
selector = "OwnerDN"
result = yield self.threadTask( RPC.getPilotStatistics, selector, req )
if not result['OK']:
if 'FromDate' in req:
del req['FromDate']
if 'LastUpdate' in req:
del req['LastUpdate']
if 'ToDate' in req:
del req['ToDate']
result = yield self.threadTask( RPC.getCounters, "PilotAgents", [selector], req )
statistics = {}
if result['OK']:
for status, count in result['Value']:
if "OwnerDN" in status:
userName = getUsernameForDN( status['OwnerDN'] )
if userName['OK']:
status['OwnerDN'] = userName['Value']
statistics[status[selector]] = count
result = S_OK(statistics)
if result["OK"]:
callback = []
result = dict( result["Value"] )
keylist = result.keys()
keylist.sort()
if selector == "Site":
tier1 = gConfig.getValue( "/WebApp/PreferredSites", [] )
if len( tier1 ) > 0:
tier1.sort()
for i in tier1:
if result.has_key( i ):
countryCode = i.rsplit( ".", 1 )[1]
callback.append( {"key":i, "value":result[i], "code":countryCode, "color": paletteColor.getColor( countryCode ) } )
for key in keylist:
if selector == "Site" and tier1:
if key not in tier1:
try:
countryCode = key.rsplit( ".", 1 )[1]
except:
countryCode = "Unknown"
callback.append( {"key":key, "value":result[key], "code":countryCode, "color": paletteColor.getColor( key ) } )
elif selector == "Site" and not tier1:
try:
countryCode = key.rsplit( ".", 1 )[1]
except:
countryCode = "Unknown"
callback.append( {"key":key, "value":result[key], "code":countryCode, "color": paletteColor.getColor( key ) } )
else:
callback.append( {"key":key, "value":result[key], "code":"", "color": paletteColor.getColor( key ) } )
callback = {"success":"true", "result":callback}
else:
callback = {"success":"false", "error":result["Message"]}
self.finish( callback )
def getUsersGroupsAndSEs( queryQueue, name_id_se ):
""" This dumps the users, groups and SEs from the LFC, converts them into DIRAC group, and
create the query to insert them into the DFC.
WATCH OUT !!
The DIRAC group is evaluated from the VOMS attribute, since this is what is used in the LFC.
So if you have several DIRAC groups that have the same voms role, the assigned group will
be the first one...
:param queryQueue : queue in which to put the SQL statement to be executed against the DFC
:param name_id_se : cache to be filled in with the mapping {seName:id in the DFC}
"""
connection = cx_Oracle.Connection( prodDbAccount, prodDbAccountPassword, prodDbTNS, threaded = True )
fromdbR_ = connection.cursor()
fromdbR_.arraysize = 1000
# First, fetch all the UID and DN from the LFC
if fromdbR_.execute( "SELECT USERID, USERNAME from CNS_USERINFO" ):
rows = fromdbR_.fetchall()
for row in rows:
uid = row[0]
dn = row[1]
name = uid_name.get( uid )
# If the name is not in cache,
if not name:
# We do some mangling of the DN to try to have sensitive name whatever happen
# We also include the UID in case the name is not unique, or several DN are associated
namePart = 'name'
idPart = ''
dnSplit = dn.split( 'CN=' )
try:
namePart = dnSplit[-1].replace( "'", " " )
except Exception as e:
pass
try:
idPart = dnSplit[2].replace( "/", "" )
except Exception as e:
pass
idPart += ' uid_%s' % uid
name = "Unknown (%s %s)" % ( namePart, idPart )
# Now, we do try to convert the DN into a DIRAC username
if "Unknown" in name:
result = getUsernameForDN( dn )
if result['OK']:
name = result['Value']
uid_name[uid] = name
# Now we prepare the SQL statement to insert them into the DFC
for uid in uid_name:
username = uid_name[uid]
queryQueue.put( "INSERT INTO FC_Users(UID, UserName) values (%s, '%s');\n" % ( uid, username ) )
# Now, same principle on the group
if fromdbR_.execute( "SELECT GID, GROUPNAME from CNS_GROUPINFO" ):
rows = fromdbR_.fetchall()
for row in rows:
gid = row[0]
gn = row[1]
groupname = gn
# CAUTION: as explained in the docstring, if multiple groups share the same voms role
# we take the first one
groups = getGroupsWithVOMSAttribute( '/' + gn )
if groups:
groupname = groups[0]
queryQueue.put( "INSERT INTO FC_Groups(GID, GroupName) values (%s, '%s');\n" % ( gid, groupname ) )
# We build a cache that contains the mapping between the name and its ID in the DFC DB
# The ID starts at 2 because ID=1 is taken by FakeSe
seCounter = 2
# Fetch the name from the LFC
if fromdbR_.execute( "select unique HOST from CNS_FILE_REPLICA" ):
rows = fromdbR_.fetchall()
for row in rows:
seName = row[0]
# Populate the SE cache
name_id_se[seName] = seCounter
# Create the query for the DFC
queryQueue.put( "INSERT INTO FC_StorageElements(SEID, SEName) values (%s, '%s');\n" % ( seCounter, seName ) )
seCounter += 1
# Also here we just insert all the statuses defined above
for status in dirac_status:
queryQueue.put( "INSERT INTO FC_Statuses (Status) values ('%s');\n" % status )
fromdbR_.close()
connection.close()
# Set the poison pill in the queue
queryQueue.put( None )
return
def web_getStatisticsData(self):
req = self.__request()
paletteColor = Palette()
RPC = RPCClient("WorkloadManagement/WMSAdministrator")
selector = self.request.arguments["statsField"][0]
if selector == 'Site':
selector = "GridSite"
if selector == "Computing Element":
selector = "DestinationSite"
elif selector == "Owner Group":
selector = "OwnerGroup"
elif selector == "Owner":
selector = "OwnerDN"
result = yield self.threadTask(RPC.getPilotStatistics, selector, req)
if not result['OK']:
if 'FromDate' in req:
del req['FromDate']
if 'LastUpdate' in req:
del req['LastUpdate']
if 'ToDate' in req:
del req['ToDate']
result = yield self.threadTask(RPC.getCounters, "PilotAgents",
[selector], req)
statistics = {}
if result['OK']:
for status, count in result['Value']:
if "OwnerDN" in status:
userName = getUsernameForDN(status['OwnerDN'])
if userName['OK']:
status['OwnerDN'] = userName['Value']
statistics[status[selector]] = count
result = S_OK(statistics)
if result["OK"]:
callback = []
result = dict(result["Value"])
keylist = result.keys()
keylist.sort()
if selector == "Site":
tier1 = gConfig.getValue("/WebApp/PreferredSites", [])
if len(tier1) > 0:
tier1.sort()
for i in tier1:
if result.has_key(i):
countryCode = i.rsplit(".", 1)[1]
callback.append({
"key":
i,
"value":
result[i],
"code":
countryCode,
"color":
paletteColor.getColor(countryCode)
})
for key in keylist:
if selector == "Site" and tier1:
if key not in tier1:
try:
countryCode = key.rsplit(".", 1)[1]
except:
countryCode = "Unknown"
callback.append({
"key": key,
"value": result[key],
"code": countryCode,
"color": paletteColor.getColor(key)
})
elif selector == "Site" and not tier1:
try:
countryCode = key.rsplit(".", 1)[1]
except:
countryCode = "Unknown"
callback.append({
"key": key,
"value": result[key],
"code": countryCode,
"color": paletteColor.getColor(key)
})
else:
callback.append({
"key": key,
"value": result[key],
"code": "",
"color": paletteColor.getColor(key)
})
callback = {"success": "true", "result": callback}
else:
callback = {"success": "false", "error": result["Message"]}
self.finish(callback)
def getUsersGroupsAndSEs(queryQueue, name_id_se):
""" This dumps the users, groups and SEs from the LFC, converts them into DIRAC group, and
create the query to insert them into the DFC.
WATCH OUT !!
The DIRAC group is evaluated from the VOMS attribute, since this is what is used in the LFC.
So if you have several DIRAC groups that have the same voms role, the assigned group will
be the first one...
:param queryQueue : queue in which to put the SQL statement to be executed against the DFC
:param name_id_se : cache to be filled in with the mapping {seName:id in the DFC}
"""
connection = cx_Oracle.Connection(prodDbAccount,
prodDbAccountPassword,
prodDbTNS,
threaded=True)
fromdbR_ = connection.cursor()
fromdbR_.arraysize = 1000
# First, fetch all the UID and DN from the LFC
if fromdbR_.execute("SELECT USERID, USERNAME from CNS_USERINFO"):
rows = fromdbR_.fetchall()
for row in rows:
uid = row[0]
dn = row[1]
name = uid_name.get(uid)
# If the name is not in cache,
if not name:
# We do some mangling of the DN to try to have sensitive name whatever happen
# We also include the UID in case the name is not unique, or several DN are associated
namePart = 'name'
idPart = ''
dnSplit = dn.split('CN=')
try:
namePart = dnSplit[-1].replace("'", " ")
except Exception as e:
pass
try:
idPart = dnSplit[2].replace("/", "")
except Exception as e:
pass
idPart += ' uid_%s' % uid
name = "Unknown (%s %s)" % (namePart, idPart)
# Now, we do try to convert the DN into a DIRAC username
if "Unknown" in name:
result = getUsernameForDN(dn)
if result['OK']:
name = result['Value']
uid_name[uid] = name
# Now we prepare the SQL statement to insert them into the DFC
for uid in uid_name:
username = uid_name[uid]
queryQueue.put(
"INSERT INTO FC_Users(UID, UserName) values (%s, '%s');\n" %
(uid, username))
# Now, same principle on the group
if fromdbR_.execute("SELECT GID, GROUPNAME from CNS_GROUPINFO"):
rows = fromdbR_.fetchall()
for row in rows:
gid = row[0]
gn = row[1]
groupname = gn
# CAUTION: as explained in the docstring, if multiple groups share the same voms role
# we take the first one
groups = getGroupsWithVOMSAttribute('/' + gn)
if groups:
groupname = groups[0]
queryQueue.put(
"INSERT INTO FC_Groups(GID, GroupName) values (%s, '%s');\n" %
(gid, groupname))
# We build a cache that contains the mapping between the name and its ID in the DFC DB
# The ID starts at 2 because ID=1 is taken by FakeSe
seCounter = 2
# Fetch the name from the LFC
if fromdbR_.execute("select unique HOST from CNS_FILE_REPLICA"):
rows = fromdbR_.fetchall()
for row in rows:
seName = row[0]
# Populate the SE cache
name_id_se[seName] = seCounter
# Create the query for the DFC
queryQueue.put(
"INSERT INTO FC_StorageElements(SEID, SEName) values (%s, '%s');\n"
% (seCounter, seName))
seCounter += 1
# Also here we just insert all the statuses defined above
for status in dirac_status:
queryQueue.put("INSERT INTO FC_Statuses (Status) values ('%s');\n" %
status)
fromdbR_.close()
connection.close()
# Set the poison pill in the queue
queryQueue.put(None)
return
def main():
global verbose
global taskQueueID
Script.registerSwitch("v", "verbose", "give max details about task queues",
setVerbose)
Script.registerSwitch("t:", "taskQueue=", "show this task queue only",
setTaskQueueID)
Script.parseCommandLine(initializeMonitor=False)
result = MatcherClient().getActiveTaskQueues()
if not result['OK']:
gLogger.error(result['Message'])
sys.exit(1)
tqDict = result['Value']
if not verbose:
fields = [
'TaskQueue', 'Jobs', 'CPUTime', 'Owner', 'OwnerGroup', 'Sites',
'Platforms', 'SubmitPools', 'Setup', 'Priority'
]
records = []
for tqId in sorted(tqDict):
if taskQueueID and tqId != taskQueueID:
continue
record = [str(tqId)]
tqData = tqDict[tqId]
for key in fields[1:]:
if key == 'Owner':
value = tqData.get('OwnerDN', '-')
if value != '-':
result = getUsernameForDN(value)
if not result['OK']:
value = 'Unknown'
else:
value = result['Value']
else:
value = tqData.get(key, '-')
if isinstance(value, list):
if len(value) > 1:
record.append(str(value[0]) + '...')
else:
record.append(str(value[0]))
else:
record.append(str(value))
records.append(record)
printTable(fields, records)
else:
fields = ['Key', 'Value']
for tqId in sorted(tqDict):
if taskQueueID and tqId != taskQueueID:
continue
gLogger.notice("\n==> TQ %s" % tqId)
records = []
tqData = tqDict[tqId]
for key in sorted(tqData):
value = tqData[key]
if isinstance(value, list):
records.append([key, {"Value": value, 'Just': 'L'}])
else:
value = str(value)
records.append([key, {"Value": value, 'Just': 'L'}])
printTable(fields, records, numbering=False)
if not verbose:
fields = ['TaskQueue','Jobs','CPUTime','Owner','OwnerGroup','Sites',
'Platforms','SubmitPools','Setup','Priority']
records = []
print
for tqId in sorted( tqDict ):
if taskQueueID and tqId != taskQueueID:
continue
record = [str(tqId)]
tqData = tqDict[ tqId ]
for key in fields[1:]:
if key == 'Owner':
value = tqData.get( 'OwnerDN', '-' )
if value != '-':
result = getUsernameForDN( value )
if not result['OK']:
value = 'Unknown'
else:
value = result['Value']
else:
value = tqData.get( key, '-' )
if type( value ) == types.ListType:
if len( value ) > 1:
record.append( str( value[0] ) + '...' )
else:
record.append( str( value[0] ) )
else:
record.append( str( value ) )
records.append( record )
def generateProxyOrToken(
self, client, grant_type, user=None, scope=None, expires_in=None, include_refresh_token=True
):
"""Generate proxy or tokens after authorization
:param client: instance of the IdP client
:param grant_type: authorization grant type (unused)
:param str user: user identificator
:param str scope: requested scope
:param expires_in: when the token should expire (unused)
:param bool include_refresh_token: to include refresh token (unused)
:return: dict or str -- will return tokens as dict or proxy as string
"""
# Read requested scopes
group = self._getScope(scope, "g")
lifetime = self._getScope(scope, "lifetime")
# Found provider name for group
provider = getIdPForGroup(group)
# Search DIRAC username by user ID
result = getUsernameForDN(wrapIDAsDN(user))
if not result["OK"]:
raise OAuth2Error(result["Message"])
userName = result["Value"]
# User request a proxy
if "proxy" in scope_to_list(scope):
# Try to return user proxy if proxy scope present in the authorization request
if not isDownloadProxyAllowed():
raise OAuth2Error("You can't get proxy, configuration(allowProxyDownload) not allow to do that.")
sLog.debug(
"Try to query %s@%s proxy%s" % (user, group, ("with lifetime:%s" % lifetime) if lifetime else "")
)
# Get user DNs
result = getDNForUsername(userName)
if not result["OK"]:
raise OAuth2Error(result["Message"])
userDNs = result["Value"]
err = []
# Try every DN to generate a proxy
for dn in userDNs:
sLog.debug("Try to get proxy for %s" % dn)
params = {}
if lifetime:
params["requiredTimeLeft"] = int(lifetime)
# if the configuration describes adding a VOMS extension, we will do so
if getGroupOption(group, "AutoAddVOMS", False):
result = self.proxyCli.downloadVOMSProxy(dn, group, **params)
else:
# otherwise we will return the usual proxy
result = self.proxyCli.downloadProxy(dn, group, **params)
if not result["OK"]:
err.append(result["Message"])
else:
sLog.info("Proxy was created.")
result = result["Value"].dumpAllToString()
if not result["OK"]:
raise OAuth2Error(result["Message"])
# Proxy generated
return {
"proxy": result["Value"].decode() if isinstance(result["Value"], bytes) else result["Value"]
}
# Proxy cannot be generated or not found
raise OAuth2Error("; ".join(err))
# User request a tokens
else:
# Ask TokenManager to generate new tokens for user
result = self.tokenCli.getToken(userName, group)
if not result["OK"]:
raise OAuth2Error(result["Message"])
token = result["Value"]
# Wrap the refresh token and register it to protect against reuse
result = self.registerRefreshToken(
dict(sub=user, scope=scope, provider=provider, azp=client.get_client_id()), token
)
if not result["OK"]:
raise OAuth2Error(result["Message"])
# Return tokens as dictionary
return result["Value"]
fields = [
'TaskQueue', 'Jobs', 'CPUTime', 'Owner', 'OwnerGroup', 'Sites',
'Platforms', 'SubmitPools', 'Setup', 'Priority'
]
records = []
for tqId in sorted(tqDict):
if taskQueueID and tqId != taskQueueID:
continue
record = [str(tqId)]
tqData = tqDict[tqId]
for key in fields[1:]:
if key == 'Owner':
value = tqData.get('OwnerDN', '-')
if value != '-':
result = getUsernameForDN(value)
if not result['OK']:
value = 'Unknown'
else:
value = result['Value']
else:
value = tqData.get(key, '-')
if isinstance(value, list):
if len(value) > 1:
record.append(str(value[0]) + '...')
else:
record.append(str(value[0]))
else:
record.append(str(value))
records.append(record)
def main():
global verbose
global taskQueueID
Script.registerSwitch("v", "verbose", "give max details about task queues",
setVerbose)
Script.registerSwitch("t:", "taskQueue=", "show this task queue only",
setTaskQueueID)
Script.parseCommandLine(initializeMonitor=False)
result = MatcherClient().getActiveTaskQueues()
if not result["OK"]:
gLogger.error(result["Message"])
sys.exit(1)
tqDict = result["Value"]
if not verbose:
fields = [
"TaskQueue",
"Jobs",
"CPUTime",
"Owner",
"OwnerGroup",
"Sites",
"Platforms",
"Priority",
]
records = []
for tqId in sorted(tqDict):
if taskQueueID and tqId != taskQueueID:
continue
record = [str(tqId)]
tqData = tqDict[tqId]
for key in fields[1:]:
if key == "Owner":
value = tqData.get("OwnerDN", "-")
if value != "-":
result = getUsernameForDN(value)
if not result["OK"]:
value = "Unknown"
else:
value = result["Value"]
else:
value = tqData.get(key, "-")
if isinstance(value, list):
if len(value) > 1:
record.append(str(value[0]) + "...")
else:
record.append(str(value[0]))
else:
record.append(str(value))
records.append(record)
printTable(fields, records)
else:
fields = ["Key", "Value"]
for tqId in sorted(tqDict):
if taskQueueID and tqId != taskQueueID:
continue
gLogger.notice("\n==> TQ %s" % tqId)
records = []
tqData = tqDict[tqId]
for key in sorted(tqData):
value = tqData[key]
if isinstance(value, list):
records.append([key, {"Value": value, "Just": "L"}])
else:
value = str(value)
records.append([key, {"Value": value, "Just": "L"}])
printTable(fields, records, numbering=False)
