def getGroupScopes(self, group: str) -> list:
"""Get group scopes
:param group: DIRAC group
"""
idPScope = getGroupOption(group, "IdPRole")
return scope_to_list(idPScope) if idPScope else []
def export_getPilotLoggingInfo(self, pilotReference):
""" Get the pilot logging info for the Grid job reference
"""
result = pilotDB.getPilotInfo(pilotReference)
if not result['OK'] or not result['Value']:
return S_ERROR('Failed to determine owner for pilot ' +
pilotReference)
pilotDict = result['Value'][pilotReference]
owner = pilotDict['OwnerDN']
group = pilotDict['OwnerGroup']
group = getGroupOption(group, 'VOMSRole', group)
ret = gProxyManager.getPilotProxyFromVOMSGroup(owner, group)
if not ret['OK']:
gLogger.error(ret['Message'])
gLogger.error('Could not get proxy:',
'User "%s", Group "%s"' % (owner, group))
return S_ERROR("Failed to get the pilot's owner proxy")
proxy = ret['Value']
gridType = pilotDict['GridType']
return getPilotLoggingInfo(proxy, gridType, pilotReference)
def export_killPilot(self, pilotRefList ):
""" Kill the specified pilots
"""
# Make a list if it is not yet
pilotRefs = list( pilotRefList )
if type( pilotRefList ) in StringTypes:
pilotRefs = [pilotRefList]
# Regroup pilots per site and per owner
pilotRefDict = {}
for pilotReference in pilotRefs:
result = pilotDB.getPilotInfo(pilotReference)
if not result['OK'] or not result[ 'Value' ]:
return S_ERROR('Failed to get info for pilot ' + pilotReference)
pilotDict = result['Value'][pilotReference]
owner = pilotDict['OwnerDN']
group = pilotDict['OwnerGroup']
queue = '@@@'.join( [owner, group, pilotDict['GridSite'], pilotDict['DestinationSite'], pilotDict['Queue']] )
gridType = pilotDict['GridType']
pilotRefDict.setdefault( queue, {} )
pilotRefDict[queue].setdefault( 'PilotList', [] )
pilotRefDict[queue]['PilotList'].append( pilotReference )
pilotRefDict[queue]['GridType'] = gridType
# Do the work now queue by queue
ceFactory = ComputingElementFactory()
failed = []
for key, pilotDict in pilotRefDict.items():
owner,group,site,ce,queue = key.split( '@@@' )
result = getQueue( site, ce, queue )
if not result['OK']:
return result
queueDict = result['Value']
gridType = pilotDict['GridType']
result = ceFactory.getCE( gridType, ce, queueDict )
if not result['OK']:
return result
ce = result['Value']
if gridType in ["LCG","gLite","CREAM"]:
group = getGroupOption(group,'VOMSRole',group)
ret = gProxyManager.getPilotProxyFromVOMSGroup( owner, group )
if not ret['OK']:
gLogger.error( ret['Message'] )
gLogger.error( 'Could not get proxy:', 'User "%s", Group "%s"' % ( owner, group ) )
return S_ERROR("Failed to get the pilot's owner proxy")
proxy = ret['Value']
ce.setProxy( proxy )
pilotList = pilotDict['PilotList']
result = ce.killJob( pilotList )
if not result['OK']:
failed.extend( pilotList )
if failed:
return S_ERROR('Failed to kill at least some pilots')
return S_OK()
def __getGridJobOutput(self,pilotReference):
""" Get the pilot job standard output and standard error files for the Grid
job reference
"""
result = pilotDB.getPilotInfo(pilotReference)
if not result['OK'] or not result[ 'Value' ]:
return S_ERROR('Failed to get info for pilot ' + pilotReference)
pilotDict = result['Value'][pilotReference]
owner = pilotDict['OwnerDN']
group = pilotDict['OwnerGroup']
# FIXME: What if the OutputSandBox is not StdOut and StdErr, what do we do with other files?
result = pilotDB.getPilotOutput(pilotReference)
if result['OK']:
stdout = result['Value']['StdOut']
error = result['Value']['StdErr']
if stdout or error:
resultDict = {}
resultDict['StdOut'] = stdout
resultDict['StdErr'] = error
resultDict['OwnerDN'] = owner
resultDict['OwnerGroup'] = group
resultDict['FileList'] = []
return S_OK(resultDict)
else:
return S_ERROR('Empty pilot output found')
gridType = pilotDict['GridType']
if gridType in ["LCG","gLite","CREAM"]:
group = getGroupOption(group,'VOMSRole',group)
ret = gProxyManager.getPilotProxyFromVOMSGroup( owner, group )
if not ret['OK']:
gLogger.error( ret['Message'] )
gLogger.error( 'Could not get proxy:', 'User "%s", Group "%s"' % ( owner, group ) )
return S_ERROR("Failed to get the pilot's owner proxy")
proxy = ret['Value']
pilotStamp = pilotDict['PilotStamp']
result = getPilotOutput( proxy, gridType, pilotReference, pilotStamp )
if not result['OK']:
return S_ERROR('Failed to get pilot output: '+result['Message'])
# FIXME: What if the OutputSandBox is not StdOut and StdErr, what do we do with other files?
stdout = result['StdOut']
error = result['StdErr']
fileList = result['FileList']
result = pilotDB.storePilotOutput(pilotReference,stdout,error)
if not result['OK']:
gLogger.error('Failed to store pilot output:',result['Message'])
resultDict = {}
resultDict['StdOut'] = stdout
resultDict['StdErr'] = error
resultDict['OwnerDN'] = owner
resultDict['OwnerGroup'] = group
resultDict['FileList'] = fileList
return S_OK(resultDict)
else:
return S_ERROR('Can not retrieve pilot output for the Grid %s ' % gridType)
def __buildRolesAndGroups(self):
"""Rebuild the cache dictionary for VOMS roles and EiscatDIRAC Groups"""
self.lastBuild = datetime.datetime.now()
allGroups = getAllGroups()
for grpName in allGroups:
vomsRole = getGroupOption(grpName, "VOMSRole")
if vomsRole:
self.diracGroups[grpName] = vomsRole
self.vomsRoles.setdefault(vomsRole, []).append(grpName)
def __buildRolesAndGroups(self):
""" Rebuild the cache dictionary for VOMS roles and DIRAC Groups"""
self.lastBuild = datetime.datetime.now()
allGroups = getAllGroups()
for grpName in allGroups:
vomsRole = getGroupOption(grpName, "VOMSRole")
if vomsRole:
self.diracGroups[grpName] = vomsRole
self.vomsRoles.setdefault(vomsRole, []).append(grpName)
def getGroupScopes(self, group):
"""Get group scopes
:param str group: DIRAC group
:return: list
"""
idPScope = getGroupOption(group, "IdPRole")
if not idPScope:
idPScope = "wlcg.groups:/%s/%s" % (getVOForGroup(group),
group.split("_")[1])
return S_OK(scope_to_list(idPScope))
def getPilotProxy(pilotDict):
"""Get a proxy bound to a pilot"""
owner = pilotDict["OwnerDN"]
group = pilotDict["OwnerGroup"]
groupVOMS = getGroupOption(group, "VOMSRole", group)
result = gProxyManager.getPilotProxyFromVOMSGroup(owner, groupVOMS)
if not result["OK"]:
gLogger.error("Could not get proxy:", 'User "%s" Group "%s" : %s' % (owner, groupVOMS, result["Message"]))
return S_ERROR("Failed to get the pilot's owner proxy")
proxy = result["Value"]
return S_OK(proxy)
def getGroupScopes(self, group):
"""Get group scopes
:param str group: DIRAC group
:return: list
"""
idPScope = getGroupOption(group, "IdPRole")
if not idPScope:
idPScope = "eduperson_entitlement?value=urn:mace:egi.eu:group:%s:role=%s#aai.egi.eu" % (
getVOForGroup(group),
group.split("_")[1],
)
return S_OK(scope_to_list(idPScope))
def killPilotsInQueues(pilotRefDict):
"""kill pilots queue by queue
:params dict pilotRefDict: a dict of pilots in queues
"""
ceFactory = ComputingElementFactory()
failed = []
for key, pilotDict in pilotRefDict.items():
owner, group, site, ce, queue = key.split("@@@")
result = getQueue(site, ce, queue)
if not result["OK"]:
return result
queueDict = result["Value"]
gridType = pilotDict["GridType"]
result = ceFactory.getCE(gridType, ce, queueDict)
if not result["OK"]:
return result
ce = result["Value"]
# FIXME: quite hacky. Should be either removed, or based on some flag
if gridType in ["CREAM", "ARC", "Globus", "HTCondorCE"]:
group = getGroupOption(group, "VOMSRole", group)
ret = gProxyManager.getPilotProxyFromVOMSGroup(owner, group)
if not ret["OK"]:
gLogger.error(
"Could not get proxy:", 'User "%s" Group "%s" : %s' %
(owner, group, ret["Message"]))
return S_ERROR("Failed to get the pilot's owner proxy")
proxy = ret["Value"]
ce.setProxy(proxy)
pilotList = pilotDict["PilotList"]
result = ce.killJob(pilotList)
if not result["OK"]:
failed.extend(pilotList)
return failed
def export_getPilotLoggingInfo(self,pilotReference):
""" Get the pilot logging info for the Grid job reference
"""
result = pilotDB.getPilotInfo(pilotReference)
if not result['OK'] or not result[ 'Value' ]:
return S_ERROR('Failed to determine owner for pilot ' + pilotReference)
pilotDict = result['Value'][pilotReference]
owner = pilotDict['OwnerDN']
group = pilotDict['OwnerGroup']
group = getGroupOption(group,'VOMSRole',group)
ret = gProxyManager.getPilotProxyFromVOMSGroup( owner, group )
if not ret['OK']:
gLogger.error( ret['Message'] )
gLogger.error( 'Could not get proxy:', 'User "%s", Group "%s"' % ( owner, group ) )
return S_ERROR("Failed to get the pilot's owner proxy")
proxy = ret['Value']
gridType = pilotDict['GridType']
return getPilotLoggingInfo( proxy, gridType, pilotReference )
def __getGridJobOutput(self, pilotReference):
""" Get the pilot job standard output and standard error files for the Grid
job reference
"""
result = pilotDB.getPilotInfo(pilotReference)
if not result['OK'] or not result['Value']:
return S_ERROR('Failed to get info for pilot ' + pilotReference)
pilotDict = result['Value'][pilotReference]
owner = pilotDict['OwnerDN']
group = pilotDict['OwnerGroup']
# FIXME: What if the OutputSandBox is not StdOut and StdErr, what do we do with other files?
result = pilotDB.getPilotOutput(pilotReference)
if result['OK']:
stdout = result['Value']['StdOut']
error = result['Value']['StdErr']
if stdout or error:
resultDict = {}
resultDict['StdOut'] = stdout
resultDict['StdErr'] = error
resultDict['OwnerDN'] = owner
resultDict['OwnerGroup'] = group
resultDict['FileList'] = []
return S_OK(resultDict)
else:
return S_ERROR('Empty pilot output found')
gridType = pilotDict['GridType']
if gridType in ["LCG", "gLite", "CREAM"]:
group = getGroupOption(group, 'VOMSRole', group)
ret = gProxyManager.getPilotProxyFromVOMSGroup(owner, group)
if not ret['OK']:
gLogger.error(ret['Message'])
gLogger.error('Could not get proxy:',
'User "%s", Group "%s"' % (owner, group))
return S_ERROR("Failed to get the pilot's owner proxy")
proxy = ret['Value']
pilotStamp = pilotDict['PilotStamp']
result = getPilotOutput(proxy, gridType, pilotReference,
pilotStamp)
if not result['OK']:
return S_ERROR('Failed to get pilot output: ' +
result['Message'])
# FIXME: What if the OutputSandBox is not StdOut and StdErr, what do we do with other files?
stdout = result['StdOut']
error = result['StdErr']
fileList = result['FileList']
result = pilotDB.storePilotOutput(pilotReference, stdout, error)
if not result['OK']:
gLogger.error('Failed to store pilot output:',
result['Message'])
resultDict = {}
resultDict['StdOut'] = stdout
resultDict['StdErr'] = error
resultDict['OwnerDN'] = owner
resultDict['OwnerGroup'] = group
resultDict['FileList'] = fileList
return S_OK(resultDict)
else:
return S_ERROR('Can not retrieve pilot output for the Grid %s ' %
gridType)
def getGridJobOutput(pilotReference):
""" Get the pilot job standard output and standard error files for the Grid job reference
:param str pilotReference: a grid (job) pilot reference
"""
result = pilotAgentsDB.getPilotInfo(pilotReference)
if not result['OK'] or not result['Value']:
return S_ERROR('Failed to get info for pilot ' + pilotReference)
pilotDict = result['Value'][pilotReference]
owner = pilotDict['OwnerDN']
group = pilotDict['OwnerGroup']
# FIXME: What if the OutputSandBox is not StdOut and StdErr, what do we do with other files?
result = pilotAgentsDB.getPilotOutput(pilotReference)
if result['OK']:
stdout = result['Value']['StdOut']
error = result['Value']['StdErr']
if stdout or error:
resultDict = {}
resultDict['StdOut'] = stdout
resultDict['StdErr'] = error
resultDict['OwnerDN'] = owner
resultDict['OwnerGroup'] = group
resultDict['FileList'] = []
return S_OK(resultDict)
else:
gLogger.warn('Empty pilot output found', 'for %s' % pilotReference)
# Instantiate the appropriate CE
ceFactory = ComputingElementFactory()
result = getQueue(pilotDict['GridSite'], pilotDict['DestinationSite'],
pilotDict['Queue'])
if not result['OK']:
return result
queueDict = result['Value']
gridEnv = getGridEnv()
queueDict['GridEnv'] = gridEnv
queueDict['WorkingDirectory'] = mkdtemp()
result = ceFactory.getCE(pilotDict['GridType'],
pilotDict['DestinationSite'], queueDict)
if not result['OK']:
shutil.rmtree(queueDict['WorkingDirectory'])
return result
ce = result['Value']
groupVOMS = getGroupOption(group, 'VOMSRole', group)
result = gProxyManager.getPilotProxyFromVOMSGroup(owner, groupVOMS)
if not result['OK']:
gLogger.error(
'Could not get proxy:', 'User "%s" Group "%s" : %s' %
(owner, groupVOMS, result['Message']))
return S_ERROR("Failed to get the pilot's owner proxy")
proxy = result['Value']
ce.setProxy(proxy)
pilotStamp = pilotDict['PilotStamp']
pRef = pilotReference
if pilotStamp:
pRef = pRef + ':::' + pilotStamp
result = ce.getJobOutput(pRef)
if not result['OK']:
shutil.rmtree(queueDict['WorkingDirectory'])
return result
stdout, error = result['Value']
if stdout:
result = pilotAgentsDB.storePilotOutput(pilotReference, stdout, error)
if not result['OK']:
gLogger.error('Failed to store pilot output:', result['Message'])
resultDict = {}
resultDict['StdOut'] = stdout
resultDict['StdErr'] = error
resultDict['OwnerDN'] = owner
resultDict['OwnerGroup'] = group
resultDict['FileList'] = []
shutil.rmtree(queueDict['WorkingDirectory'])
return S_OK(resultDict)
def __getGridJobOutput(self,pilotReference):
""" Get the pilot job standard output and standard error files for the Grid
job reference
"""
result = pilotDB.getPilotInfo(pilotReference)
if not result['OK'] or not result[ 'Value' ]:
return S_ERROR('Failed to get info for pilot ' + pilotReference)
pilotDict = result['Value'][pilotReference]
owner = pilotDict['OwnerDN']
group = pilotDict['OwnerGroup']
# FIXME: What if the OutputSandBox is not StdOut and StdErr, what do we do with other files?
result = pilotDB.getPilotOutput(pilotReference)
if result['OK']:
stdout = result['Value']['StdOut']
error = result['Value']['StdErr']
if stdout or error:
resultDict = {}
resultDict['StdOut'] = stdout
resultDict['StdErr'] = error
resultDict['OwnerDN'] = owner
resultDict['OwnerGroup'] = group
resultDict['FileList'] = []
return S_OK(resultDict)
else:
gLogger.warn( 'Empty pilot output found for %s' % pilotReference )
gridType = pilotDict['GridType']
if gridType in ["LCG","gLite","CREAM"]:
group = getGroupOption(group,'VOMSRole',group)
ret = gProxyManager.getPilotProxyFromVOMSGroup( owner, group )
if not ret['OK']:
gLogger.error( ret['Message'] )
gLogger.error( 'Could not get proxy:', 'User "%s", Group "%s"' % ( owner, group ) )
return S_ERROR("Failed to get the pilot's owner proxy")
proxy = ret['Value']
pilotStamp = pilotDict['PilotStamp']
result = getPilotOutput( proxy, gridType, pilotReference, pilotStamp )
if not result['OK']:
return S_ERROR('Failed to get pilot output: '+result['Message'])
# FIXME: What if the OutputSandBox is not StdOut and StdErr, what do we do with other files?
stdout = result['StdOut']
error = result['StdErr']
fileList = result['FileList']
if stdout:
result = pilotDB.storePilotOutput(pilotReference,stdout,error)
if not result['OK']:
gLogger.error('Failed to store pilot output:',result['Message'])
resultDict = {}
resultDict['StdOut'] = stdout
resultDict['StdErr'] = error
resultDict['OwnerDN'] = owner
resultDict['OwnerGroup'] = group
resultDict['FileList'] = fileList
return S_OK(resultDict)
else:
# Instantiate the appropriate CE
ceFactory = ComputingElementFactory()
result = getQueue( pilotDict['GridSite'], pilotDict['DestinationSite'], pilotDict['Queue'] )
if not result['OK']:
return result
queueDict = result['Value']
result = ceFactory.getCE( gridType, pilotDict['DestinationSite'], queueDict )
if not result['OK']:
return result
ce = result['Value']
pilotStamp = pilotDict['PilotStamp']
pRef = pilotReference
if pilotStamp:
pRef = pRef + ':::' + pilotStamp
result = ce.getJobOutput( pRef )
if not result['OK']:
return result
stdout,error = result['Value']
if stdout:
result = pilotDB.storePilotOutput(pilotReference,stdout,error)
if not result['OK']:
gLogger.error('Failed to store pilot output:',result['Message'])
resultDict = {}
resultDict['StdOut'] = stdout
resultDict['StdErr'] = error
resultDict['OwnerDN'] = owner
resultDict['OwnerGroup'] = group
resultDict['FileList'] = []
return S_OK( resultDict )
return result
credentials = result["Value"]
# Remember a clean proxy to then upload it in step 2
proxy = copy.copy(chain)
# Create local proxy with group
self.outputFile = self.outputFile or getDefaultProxyLocation()
parameters = (self.outputFile, int(self.lifetime or 12) * 3600, self.group)
# Add a VOMS extension if the group requires it
if (result := chain.generateProxyToFile(*parameters))["OK"] and (result := self.__enableCS())["OK"]:
if not self.group and (result := findDefaultGroupForDN(credentials["DN"]))["OK"]:
self.group = result["Value"] # Use default group if user don't set it
# based on the configuration we decide whether to add VOMS extensions
if getGroupOption(self.group, "AutoAddVOMS", False):
if not (vomsAttr := getVOMSAttributeForGroup(self.group)):
print(HTML(f"<yellow>No VOMS attribute foud for {self.group}</yellow>"))
else:
vo = getVOMSVOForGroup(self.group)
if not (result := VOMS().setVOMSAttributes(self.outputFile, attribute=vomsAttr, vo=vo))["OK"]:
return S_ERROR(f"Failed adding VOMS attribute: {result['Message']}")
chain = result["Value"]
result = chain.generateProxyToFile(*parameters)
if not result["OK"]:
return S_ERROR(f"Couldn't generate proxy: {result['Message']}")
if self.enableCS:
# After creating the proxy, we can try to connect to the server
if not (result := self.__enableCS())["OK"]:
return result
def export_killPilot(self, pilotRefList):
""" Kill the specified pilots
"""
# Make a list if it is not yet
pilotRefs = list(pilotRefList)
if type(pilotRefList) in StringTypes:
pilotRefs = [pilotRefList]
# Regroup pilots per site and per owner
pilotRefDict = {}
for pilotReference in pilotRefs:
result = pilotDB.getPilotInfo(pilotReference)
if not result['OK'] or not result['Value']:
return S_ERROR('Failed to get info for pilot ' +
pilotReference)
pilotDict = result['Value'][pilotReference]
owner = pilotDict['OwnerDN']
group = pilotDict['OwnerGroup']
queue = '@@@'.join([
owner, group, pilotDict['GridSite'],
pilotDict['DestinationSite'], pilotDict['Queue']
])
gridType = pilotDict['GridType']
pilotRefDict.setdefault(queue, {})
pilotRefDict[queue].setdefault('PilotList', [])
pilotRefDict[queue]['PilotList'].append(pilotReference)
pilotRefDict[queue]['GridType'] = gridType
# Do the work now queue by queue
ceFactory = ComputingElementFactory()
failed = []
for key, pilotDict in pilotRefDict.items():
owner, group, site, ce, queue = key.split('@@@')
result = getQueue(site, ce, queue)
if not result['OK']:
return result
queueDict = result['Value']
gridType = pilotDict['GridType']
result = ceFactory.getCE(gridType, ce, queueDict)
if not result['OK']:
return result
ce = result['Value']
if gridType in ["LCG", "gLite", "CREAM"]:
group = getGroupOption(group, 'VOMSRole', group)
ret = gProxyManager.getPilotProxyFromVOMSGroup(owner, group)
if not ret['OK']:
gLogger.error(ret['Message'])
gLogger.error('Could not get proxy:',
'User "%s", Group "%s"' % (owner, group))
return S_ERROR("Failed to get the pilot's owner proxy")
proxy = ret['Value']
ce.setProxy(proxy)
pilotList = pilotDict['PilotList']
result = ce.killJob(pilotList)
if not result['OK']:
failed.extend(pilotList)
if failed:
return S_ERROR('Failed to kill at least some pilots')
return S_OK()
def __getGridJobOutput(self, pilotReference):
""" Get the pilot job standard output and standard error files for the Grid
job reference
"""
result = pilotDB.getPilotInfo(pilotReference)
if not result['OK'] or not result['Value']:
return S_ERROR('Failed to get info for pilot ' + pilotReference)
pilotDict = result['Value'][pilotReference]
owner = pilotDict['OwnerDN']
group = pilotDict['OwnerGroup']
# FIXME: What if the OutputSandBox is not StdOut and StdErr, what do we do with other files?
result = pilotDB.getPilotOutput(pilotReference)
if result['OK']:
stdout = result['Value']['StdOut']
error = result['Value']['StdErr']
if stdout or error:
resultDict = {}
resultDict['StdOut'] = stdout
resultDict['StdErr'] = error
resultDict['OwnerDN'] = owner
resultDict['OwnerGroup'] = group
resultDict['FileList'] = []
return S_OK(resultDict)
else:
gLogger.warn('Empty pilot output found for %s' %
pilotReference)
gridType = pilotDict['GridType']
if gridType in ["LCG", "gLite", "CREAM"]:
group = getGroupOption(group, 'VOMSRole', group)
ret = gProxyManager.getPilotProxyFromVOMSGroup(owner, group)
if not ret['OK']:
gLogger.error(ret['Message'])
gLogger.error('Could not get proxy:',
'User "%s", Group "%s"' % (owner, group))
return S_ERROR("Failed to get the pilot's owner proxy")
proxy = ret['Value']
pilotStamp = pilotDict['PilotStamp']
result = getPilotOutput(proxy, gridType, pilotReference,
pilotStamp)
if not result['OK']:
return S_ERROR('Failed to get pilot output: ' +
result['Message'])
# FIXME: What if the OutputSandBox is not StdOut and StdErr, what do we do with other files?
stdout = result['StdOut']
error = result['StdErr']
fileList = result['FileList']
if stdout:
result = pilotDB.storePilotOutput(pilotReference, stdout,
error)
if not result['OK']:
gLogger.error('Failed to store pilot output:',
result['Message'])
resultDict = {}
resultDict['StdOut'] = stdout
resultDict['StdErr'] = error
resultDict['OwnerDN'] = owner
resultDict['OwnerGroup'] = group
resultDict['FileList'] = fileList
return S_OK(resultDict)
else:
# Instantiate the appropriate CE
ceFactory = ComputingElementFactory()
result = getQueue(pilotDict['GridSite'],
pilotDict['DestinationSite'], pilotDict['Queue'])
if not result['OK']:
return result
queueDict = result['Value']
result = ceFactory.getCE(gridType, pilotDict['DestinationSite'],
queueDict)
if not result['OK']:
return result
ce = result['Value']
pilotStamp = pilotDict['PilotStamp']
pRef = pilotReference
if pilotStamp:
pRef = pRef + ':::' + pilotStamp
result = ce.getJobOutput(pRef)
if not result['OK']:
return result
stdout, error = result['Value']
if stdout:
result = pilotDB.storePilotOutput(pilotReference, stdout,
error)
if not result['OK']:
gLogger.error('Failed to store pilot output:',
result['Message'])
resultDict = {}
resultDict['StdOut'] = stdout
resultDict['StdErr'] = error
resultDict['OwnerDN'] = owner
resultDict['OwnerGroup'] = group
resultDict['FileList'] = []
return S_OK(resultDict)
def getJobOutput(self, jobID, _localDir=None):
""" Get the specified job standard output and error files. The output is returned
as strings.
"""
if jobID.find(':::') != -1:
pilotRef, stamp = jobID.split(':::')
else:
pilotRef = jobID
stamp = ''
if not stamp:
return S_ERROR('Pilot stamp not defined for %s' % pilotRef)
# somehow when this is called from the WMSAdministrator we don't
# get the right proxy, so we do all this stuff here now. Probably
# should be fixed in the WMSAdministrator?
# Because this function is called from the WMSAdminsitrator, the
# gridEnv that is picked up is not the one from the SiteDirector
# Definition, but from Computing/CEDefaults
result = PilotAgentsDB().getPilotInfo(pilotRef)
if not result['OK'] or not result['Value']:
return S_ERROR('Failed to determine owner for pilot ' + pilotRef)
pilotDict = result['Value'][pilotRef]
owner = pilotDict['OwnerDN']
group = getGroupOption(pilotDict['OwnerGroup'], 'VOMSRole', pilotDict['OwnerGroup'])
ret = gProxyManager.getPilotProxyFromVOMSGroup(owner, group)
if not ret['OK']:
self.log.error(ret['Message'])
self.log.error('Could not get proxy:', 'User "%s", Group "%s"' % (owner, group))
return S_ERROR("Failed to get the pilot's owner proxy")
self.proxy = ret['Value']
self.log.verbose("Getting output for: %s " % pilotRef)
cmd = ['globus-job-get-output', '-out', pilotRef]
result = executeGridCommand(self.proxy, cmd, self.gridEnv)
output = ''
if result['OK']:
if not result['Value'][0]:
output = result['Value'][1]
elif result['Value'][0] == 1 and "No such file or directory" in result['Value'][2]:
output = "Standard Output is not available on the Globus service"
else:
error = '\n'.join(result['Value'][1:])
return S_ERROR(error)
else:
return S_ERROR('Failed to retrieve output for %s' % jobID)
cmd = ['globus-job-get-output', '-err', pilotRef]
result = executeGridCommand(self.proxy, cmd, self.gridEnv)
error = ''
if result['OK']:
if not result['Value'][0]:
error = result['Value'][1]
elif result['Value'][0] == 1 and "No such file or directory" in result['Value'][2]:
error = "Standard Error is not available on the Globus service"
else:
error = '\n'.join(result['Value'][1:])
return S_ERROR(error)
else:
return S_ERROR('Failed to retrieve error for %s' % jobID)
return S_OK((output, error))
def getJobOutput( self, jobID, _localDir = None ):
""" Get the specified job standard output and error files. The output is returned
as strings.
"""
if jobID.find( ':::' ) != -1:
pilotRef, stamp = jobID.split( ':::' )
else:
pilotRef = jobID
stamp = ''
if not stamp:
return S_ERROR( 'Pilot stamp not defined for %s' % pilotRef )
## somehow when this is called from the WMSAdministrator we don't
## get the right proxy, so we do all this stuff here now. Probably
## should be fixed in the WMSAdministrator?
## Because this function is called from the WMSAdminsitrator, the
## gridEnv that is picked up is not the one from the SiteDirector
## Definition, but from Computing/CEDefaults
result = PilotAgentsDB().getPilotInfo(pilotRef)
if not result['OK'] or not result[ 'Value' ]:
return S_ERROR('Failed to determine owner for pilot ' + pilotRef)
pilotDict = result['Value'][pilotRef]
owner = pilotDict['OwnerDN']
group = getGroupOption(pilotDict['OwnerGroup'],'VOMSRole',pilotDict['OwnerGroup'])
ret = gProxyManager.getPilotProxyFromVOMSGroup( owner, group )
if not ret['OK']:
self.log.error( ret['Message'] )
self.log.error( 'Could not get proxy:', 'User "%s", Group "%s"' % ( owner, group ) )
return S_ERROR("Failed to get the pilot's owner proxy")
self.proxy = ret['Value']
self.log.verbose("Getting output for: %s " % pilotRef)
cmd = ['globus-job-get-output', '-out', pilotRef ]
result = executeGridCommand( self.proxy, cmd, self.gridEnv )
output = ''
if result['OK']:
if not result['Value'][0]:
output = result['Value'][1]
elif result['Value'][0] == 1 and "No such file or directory" in result['Value'][2]:
output = "Standard Output is not available on the Globus service"
else:
error = '\n'.join( result['Value'][1:] )
return S_ERROR( error )
else:
return S_ERROR( 'Failed to retrieve output for %s' % jobID )
cmd = ['globus-job-get-output', '-err', pilotRef ]
result = executeGridCommand( self.proxy, cmd, self.gridEnv )
error = ''
if result['OK']:
if not result['Value'][0]:
error = result['Value'][1]
elif result['Value'][0] == 1 and "No such file or directory" in result['Value'][2]:
error = "Standard Error is not available on the Globus service"
else:
error = '\n'.join( result['Value'][1:] )
return S_ERROR( error )
else:
return S_ERROR( 'Failed to retrieve error for %s' % jobID )
return S_OK( ( output, error ) )
def generateProxyOrToken(
self, client, grant_type, user=None, scope=None, expires_in=None, include_refresh_token=True
):
"""Generate proxy or tokens after authorization
:param client: instance of the IdP client
:param grant_type: authorization grant type (unused)
:param str user: user identificator
:param str scope: requested scope
:param expires_in: when the token should expire (unused)
:param bool include_refresh_token: to include refresh token (unused)
:return: dict or str -- will return tokens as dict or proxy as string
"""
# Read requested scopes
group = self._getScope(scope, "g")
lifetime = self._getScope(scope, "lifetime")
# Found provider name for group
provider = getIdPForGroup(group)
# Search DIRAC username by user ID
result = getUsernameForDN(wrapIDAsDN(user))
if not result["OK"]:
raise OAuth2Error(result["Message"])
userName = result["Value"]
# User request a proxy
if "proxy" in scope_to_list(scope):
# Try to return user proxy if proxy scope present in the authorization request
if not isDownloadProxyAllowed():
raise OAuth2Error("You can't get proxy, configuration(allowProxyDownload) not allow to do that.")
sLog.debug(
"Try to query %s@%s proxy%s" % (user, group, ("with lifetime:%s" % lifetime) if lifetime else "")
)
# Get user DNs
result = getDNForUsername(userName)
if not result["OK"]:
raise OAuth2Error(result["Message"])
userDNs = result["Value"]
err = []
# Try every DN to generate a proxy
for dn in userDNs:
sLog.debug("Try to get proxy for %s" % dn)
params = {}
if lifetime:
params["requiredTimeLeft"] = int(lifetime)
# if the configuration describes adding a VOMS extension, we will do so
if getGroupOption(group, "AutoAddVOMS", False):
result = self.proxyCli.downloadVOMSProxy(dn, group, **params)
else:
# otherwise we will return the usual proxy
result = self.proxyCli.downloadProxy(dn, group, **params)
if not result["OK"]:
err.append(result["Message"])
else:
sLog.info("Proxy was created.")
result = result["Value"].dumpAllToString()
if not result["OK"]:
raise OAuth2Error(result["Message"])
# Proxy generated
return {
"proxy": result["Value"].decode() if isinstance(result["Value"], bytes) else result["Value"]
}
# Proxy cannot be generated or not found
raise OAuth2Error("; ".join(err))
# User request a tokens
else:
# Ask TokenManager to generate new tokens for user
result = self.tokenCli.getToken(userName, group)
if not result["OK"]:
raise OAuth2Error(result["Message"])
token = result["Value"]
# Wrap the refresh token and register it to protect against reuse
result = self.registerRefreshToken(
dict(sub=user, scope=scope, provider=provider, azp=client.get_client_id()), token
)
if not result["OK"]:
raise OAuth2Error(result["Message"])
# Return tokens as dictionary
return result["Value"]
