def getProxy(userDNs, userGroup, vomsAttr, proxyFilePath):
"""do the actual download of the proxy, trying the different DNs"""
for userDN in userDNs:
if vomsAttr:
result = gProxyManager.downloadVOMSProxyToFile(
userDN,
userGroup,
requiredVOMSAttribute=vomsAttr,
filePath=proxyFilePath,
requiredTimeLeft=3600,
cacheTime=3600,
)
else:
result = gProxyManager.downloadProxyToFile(userDN,
userGroup,
filePath=proxyFilePath,
requiredTimeLeft=3600,
cacheTime=3600)
if not result["OK"]:
gLogger.error(
"Can't download %sproxy " % ("VOMS" if vomsAttr else ""),
"of '%s', group %s to file: " % (userDN, userGroup) +
result["Message"],
)
else:
return result
# If proxy not found for any DN, return an error
return S_ERROR("Can't download proxy")
def getProxy(userDNs, userGroup, vomsAttr, proxyFilePath):
""" do the actual download of the proxy, trying the different DNs
"""
for userDN in userDNs:
if vomsAttr:
result = gProxyManager.downloadVOMSProxyToFile(
userDN,
userGroup,
requiredVOMSAttribute=vomsAttr,
filePath=proxyFilePath,
requiredTimeLeft=3600,
cacheTime=3600)
else:
result = gProxyManager.downloadProxyToFile(userDN,
userGroup,
filePath=proxyFilePath,
requiredTimeLeft=3600,
cacheTime=3600)
if not result['OK']:
gLogger.warn("Can't download proxy of '%s' to file" % userDN,
result['Message'])
else:
return result
return S_ERROR("Can't download proxy")
def wrapped_fcn( *args, **kwargs ):
userName = kwargs.pop( 'proxyUserName', '' )
userDN = kwargs.pop( 'proxyUserDN', '' )
userGroup = kwargs.pop( 'proxyUserGroup', '' )
vomsFlag = kwargs.pop( 'proxyWithVOMS', True )
proxyFilePath = kwargs.pop( 'proxyFilePath', False )
if ( userName or userDN ) and userGroup:
# Setup user proxy
originalUserProxy = os.environ.get( 'X509_USER_PROXY' )
if not userDN:
result = getDNForUsername( userName )
if not result[ 'OK' ]:
return result
userDN = result[ 'Value' ][0]
vomsAttr = ''
if vomsFlag:
vomsAttr = getVOMSAttributeForGroup( userGroup )
if vomsAttr:
result = gProxyManager.downloadVOMSProxyToFile( userDN, userGroup,
requiredVOMSAttribute = vomsAttr,
filePath = proxyFilePath,
requiredTimeLeft = 3600,
cacheTime = 3600 )
else:
result = gProxyManager.downloadProxyToFile( userDN, userGroup,
filePath = proxyFilePath,
requiredTimeLeft = 3600,
cacheTime = 3600 )
if not result['OK']:
gLogger.warn( "Can't download proxy to file", result['Message'] )
return result
proxyFile = result['Value']
os.environ['X509_USER_PROXY'] = proxyFile
# Check if the caller is executing with the host certificate
useServerCertificate = gConfig.useServerCertificate()
if useServerCertificate:
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )
try:
resultFcn = fcn( *args, **kwargs )
except Exception, x:
resultFcn = S_ERROR( "Exception: %s" % str( x ) )
# Restore the default host certificate usage if necessary
if useServerCertificate:
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' )
if originalUserProxy:
os.environ['X509_USER_PROXY'] = originalUserProxy
else:
os.environ.pop( 'X509_USER_PROXY' )
return resultFcn
def wrapped_fcn(*args, **kwargs):
userName = kwargs.pop("proxyUserName", "")
userGroup = kwargs.pop("proxyUserGroup", "")
vomsFlag = kwargs.pop("proxyWithVOMS", True)
proxyFilePath = kwargs.pop("proxyFilePath", False)
if userName and userGroup:
# Setup user proxy
originalUserProxy = os.environ.get("X509_USER_PROXY")
result = getDNForUsername(userName)
if not result["OK"]:
return result
userDN = result["Value"][0]
vomsAttr = ""
if vomsFlag:
vomsAttr = getVOMSAttributeForGroup(userGroup)
if vomsAttr:
result = gProxyManager.downloadVOMSProxyToFile(
userDN,
userGroup,
requiredVOMSAttribute=vomsAttr,
filePath=proxyFilePath,
requiredTimeLeft=3600,
cacheTime=3600,
)
else:
result = gProxyManager.downloadProxyToFile(
userDN, userGroup, filePath=proxyFilePath, requiredTimeLeft=3600, cacheTime=3600
)
if not result["OK"]:
return result
proxyFile = result["Value"]
os.environ["X509_USER_PROXY"] = proxyFile
# Check if the caller is executing with the host certificate
useServerCertificate = gConfig.useServerCertificate()
if useServerCertificate:
gConfigurationData.setOptionInCFG("/DIRAC/Security/UseServerCertificate", "false")
try:
resultFcn = fcn(*args, **kwargs)
except Exception, x:
resultFcn = S_ERROR("Exception: %s" % str(x))
# Restore the default host certificate usage if necessary
if useServerCertificate:
gConfigurationData.setOptionInCFG("/DIRAC/Security/UseServerCertificate", "true")
if originalUserProxy:
os.environ["X509_USER_PROXY"] = originalUserProxy
else:
os.environ.pop("X509_USER_PROXY")
return resultFcn
def wrapped_fcn( *args, **kwargs ):
userName = kwargs.pop( 'proxyUserName', '' )
userGroup = kwargs.pop( 'proxyUserGroup', '' )
vomsFlag = kwargs.pop( 'proxyWithVOMS', True )
proxyFilePath = kwargs.pop( 'proxyFilePath', False )
if userName and userGroup:
# Setup user proxy
originalUserProxy = os.environ.get( 'X509_USER_PROXY' )
result = getDNForUsername( userName )
if not result[ 'OK' ]:
return result
userDN = result[ 'Value' ][0]
vomsAttr = ''
if vomsFlag:
vomsAttr = getVOMSAttributeForGroup( userGroup )
if vomsAttr:
result = gProxyManager.downloadVOMSProxyToFile( userDN, userGroup,
requiredVOMSAttribute = vomsAttr,
filePath = proxyFilePath,
requiredTimeLeft = 3600,
cacheTime = 3600 )
else:
result = gProxyManager.downloadProxyToFile( userDN, userGroup,
filePath = proxyFilePath,
requiredTimeLeft = 3600,
cacheTime = 3600 )
if not result['OK']:
return result
proxyFile = result['Value']
os.environ['X509_USER_PROXY'] = proxyFile
# Check if the caller is executing with the host certificate
useServerCertificate = gConfig.useServerCertificate()
if useServerCertificate:
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )
try:
resultFcn = fcn( *args, **kwargs )
except Exception, x:
resultFcn = S_ERROR( "Exception: %s" % str( x ) )
# Restore the default host certificate usage if necessary
if useServerCertificate:
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' )
if originalUserProxy:
os.environ['X509_USER_PROXY'] = originalUserProxy
else:
os.environ.pop( 'X509_USER_PROXY' )
return resultFcn
def __setupManagerProxies(self):
""" setup grid proxy for all defined managers """
oHelper = Operations()
shifters = oHelper.getSections("Shifter")
if not shifters["OK"]:
self.log.error(shifters["Message"])
return shifters
shifters = shifters["Value"]
for shifter in shifters:
shifterDict = oHelper.getOptionsDict("Shifter/%s" % shifter)
if not shifterDict["OK"]:
self.log.error(shifterDict["Message"])
continue
userName = shifterDict["Value"].get("User", "")
userGroup = shifterDict["Value"].get("Group", "")
userDN = CS.getDNForUsername(userName)
if not userDN["OK"]:
self.log.error(userDN["Message"])
continue
userDN = userDN["Value"][0]
vomsAttr = CS.getVOMSAttributeForGroup(userGroup)
if vomsAttr:
self.log.debug(
"getting VOMS [%s] proxy for shifter %s@%s (%s)" %
(vomsAttr, userName, userGroup, userDN))
getProxy = gProxyManager.downloadVOMSProxyToFile(
userDN,
userGroup,
requiredTimeLeft=1200,
cacheTime=4 * 43200)
else:
self.log.debug("getting proxy for shifter %s@%s (%s)" %
(userName, userGroup, userDN))
getProxy = gProxyManager.downloadProxyToFile(
userDN,
userGroup,
requiredTimeLeft=1200,
cacheTime=4 * 43200)
if not getProxy["OK"]:
self.log.error(getProxy["Message"])
return S_ERROR("unable to setup shifter proxy for %s: %s" %
(shifter, getProxy["Message"]))
chain = getProxy["chain"]
fileName = getProxy["Value"]
self.log.debug("got %s: %s %s" % (shifter, userName, userGroup))
self.__managersDict[shifter] = {
"ShifterDN": userDN,
"ShifterName": userName,
"ShifterGroup": userGroup,
"Chain": chain,
"ProxyFile": fileName
}
return S_OK()
def getShifterProxy(shifterType, fileName=False):
"""
This method returns a shifter's proxy
:param shifterType: ProductionManager / DataManager...
"""
if fileName:
try:
os.makedirs(os.path.dirname(fileName))
except OSError:
pass
opsHelper = Operations()
userName = opsHelper.getValue(cfgPath('Shifter', shifterType, 'User'), '')
if not userName:
return S_ERROR("No shifter User defined for %s" % shifterType)
result = CS.getDNForUsername(userName)
if not result['OK']:
return result
userDN = result['Value'][0]
result = CS.findDefaultGroupForDN(userDN)
if not result['OK']:
return result
defaultGroup = result['Value']
userGroup = opsHelper.getValue(cfgPath('Shifter', shifterType, 'Group'),
defaultGroup)
vomsAttr = CS.getVOMSAttributeForGroup(userGroup)
if vomsAttr:
gLogger.info("Getting VOMS [%s] proxy for shifter %s@%s (%s)" %
(vomsAttr, userName, userGroup, userDN))
result = gProxyManager.downloadVOMSProxyToFile(userDN,
userGroup,
filePath=fileName,
requiredTimeLeft=86400,
cacheTime=86400)
else:
gLogger.info("Getting proxy for shifter %s@%s (%s)" %
(userName, userGroup, userDN))
result = gProxyManager.downloadProxyToFile(userDN,
userGroup,
filePath=fileName,
requiredTimeLeft=86400,
cacheTime=86400)
if not result['OK']:
return result
chain = result['chain']
fileName = result['Value']
return S_OK({
'DN': userDN,
'username': userName,
'group': userGroup,
'chain': chain,
'proxyFile': fileName
})
def getShifterProxy(shifterType, fileName=False):
"""This method returns a shifter's proxy
:param str shifterType: ProductionManager / DataManager...
:param str fileName: file name
:return: S_OK(dict)/S_ERROR()
"""
if fileName:
mkDir(os.path.dirname(fileName))
opsHelper = Operations()
userName = opsHelper.getValue(cfgPath("Shifter", shifterType, "User"), "")
if not userName:
return S_ERROR("No shifter User defined for %s" % shifterType)
result = Registry.getDNForUsername(userName)
if not result["OK"]:
return result
userDN = result["Value"][0]
result = Registry.findDefaultGroupForDN(userDN)
if not result["OK"]:
return result
defaultGroup = result["Value"]
userGroup = opsHelper.getValue(cfgPath("Shifter", shifterType, "Group"),
defaultGroup)
vomsAttr = Registry.getVOMSAttributeForGroup(userGroup)
if vomsAttr:
gLogger.info("Getting VOMS [%s] proxy for shifter %s@%s (%s)" %
(vomsAttr, userName, userGroup, userDN))
result = gProxyManager.downloadVOMSProxyToFile(userDN,
userGroup,
filePath=fileName,
requiredTimeLeft=86400,
cacheTime=86400)
else:
gLogger.info("Getting proxy for shifter %s@%s (%s)" %
(userName, userGroup, userDN))
result = gProxyManager.downloadProxyToFile(userDN,
userGroup,
filePath=fileName,
requiredTimeLeft=86400,
cacheTime=86400)
if not result["OK"]:
return result
chain = result["chain"]
fileName = result["Value"]
return S_OK({
"DN": userDN,
"username": userName,
"group": userGroup,
"chain": chain,
"proxyFile": fileName
})
def getShifterProxy( shifterType, fileName = False ):
"""
This method returns a shifter's proxy
:param shifterType: ProductionManager / DataManager...
"""
if fileName:
try:
os.makedirs( os.path.dirname( fileName ) )
except OSError:
pass
opsHelper = Operations()
userName = opsHelper.getValue( cfgPath( 'Shifter', shifterType, 'User' ), '' )
if not userName:
return S_ERROR( "No shifter User defined for %s" % shifterType )
result = CS.getDNForUsername( userName )
if not result[ 'OK' ]:
return result
userDN = result[ 'Value' ][0]
result = CS.findDefaultGroupForDN( userDN )
if not result['OK']:
return result
defaultGroup = result['Value']
userGroup = opsHelper.getValue( cfgPath( 'Shifter', shifterType, 'Group' ), defaultGroup )
vomsAttr = CS.getVOMSAttributeForGroup( userGroup )
if vomsAttr:
gLogger.info( "Getting VOMS [%s] proxy for shifter %s@%s (%s)" % ( vomsAttr, userName,
userGroup, userDN ) )
result = gProxyManager.downloadVOMSProxyToFile( userDN, userGroup,
filePath = fileName,
requiredTimeLeft = 86400,
cacheTime = 86400 )
else:
gLogger.info( "Getting proxy for shifter %s@%s (%s)" % ( userName, userGroup, userDN ) )
result = gProxyManager.downloadProxyToFile( userDN, userGroup,
filePath = fileName,
requiredTimeLeft = 86400,
cacheTime = 86400 )
if not result[ 'OK' ]:
return result
chain = result[ 'chain' ]
fileName = result[ 'Value' ]
return S_OK( { 'DN' : userDN,
'username' : userName,
'group' : userGroup,
'chain' : chain,
'proxyFile' : fileName } )
def __setupManagerProxies( self ):
""" setup grid proxy for all defined managers """
oHelper = Operations()
shifters = oHelper.getSections( "Shifter" )
if not shifters["OK"]:
self.log.error( shifters["Message"] )
return shifters
shifters = shifters["Value"]
for shifter in shifters:
shifterDict = oHelper.getOptionsDict( "Shifter/%s" % shifter )
if not shifterDict["OK"]:
self.log.error( shifterDict["Message"] )
continue
userName = shifterDict["Value"].get( "User", "" )
userGroup = shifterDict["Value"].get( "Group", "" )
userDN = CS.getDNForUsername( userName )
if not userDN["OK"]:
self.log.error( userDN["Message"] )
continue
userDN = userDN["Value"][0]
vomsAttr = CS.getVOMSAttributeForGroup( userGroup )
if vomsAttr:
self.log.debug( "getting VOMS [%s] proxy for shifter %s@%s (%s)" % ( vomsAttr, userName,
userGroup, userDN ) )
getProxy = gProxyManager.downloadVOMSProxyToFile( userDN, userGroup,
requiredTimeLeft = 1200,
cacheTime = 4 * 43200 )
else:
self.log.debug( "getting proxy for shifter %s@%s (%s)" % ( userName, userGroup, userDN ) )
getProxy = gProxyManager.downloadProxyToFile( userDN, userGroup,
requiredTimeLeft = 1200,
cacheTime = 4 * 43200 )
if not getProxy["OK"]:
self.log.error( getProxy["Message" ] )
return S_ERROR( "unable to setup shifter proxy for %s: %s" % ( shifter, getProxy["Message"] ) )
chain = getProxy["chain"]
fileName = getProxy["Value" ]
self.log.debug( "got %s: %s %s" % ( shifter, userName, userGroup ) )
self.__managersDict[shifter] = { "ShifterDN" : userDN,
"ShifterName" : userName,
"ShifterGroup" : userGroup,
"Chain" : chain,
"ProxyFile" : fileName }
return S_OK()
def export_execWizardScript(self, wizard, wizpar):
""" Execure wizard with parameters """
creds = self.__clientCredentials()
if creds['Group'] != 'lhcb_prmgr':
# return S_ERROR("You have to be production manager")
if 'Generate' in wizpar:
del wizpar['Generate']
result = gProxyManager.downloadProxyToFile(creds['DN'], creds['Group'],
filePath=False,
requiredTimeLeft=86400,
cacheTime=86400)
if not result['OK']:
return result
proxyFile = result['Value']['proxyFile']
try:
f = tempfile.mkstemp()
os.write(f[0], "wizardParameters = {\n")
for name, value in wizpar.items():
os.write(f[0], " \"" + str(name) + "\": \"\"\"" + str(value) + "\"\"\",\n")
os.write(f[0], "}\n")
os.write(f[0], wizard)
os.close(f[0])
except Exception as msg:
gLogger.error("In temporary files createion: " + str(msg))
os.remove(proxyFile)
return S_ERROR(str(msg))
setenv = "source /opt/dirac/bashrc"
# #proxy = "X509_USER_PROXY=xxx"
proxy = "X509_USER_PROXY=%s" % proxyFile
cmd = "python %s" % (f[1])
try:
res = shellCall(1800, ["/bin/bash -c '%s;%s %s'"
% (setenv, proxy, cmd)])
if res['OK']:
result = S_OK(str(res['Value'][1]) + str(res['Value'][2]))
else:
gLogger.error(res['Message'])
result = res
except Exception as msg:
gLogger.error("During execution: " + str(msg))
result = S_ERROR("Failed to execute: %s" % str(msg))
os.remove(f[1])
os.remove(proxyFile)
return result
def submitNewBigJob( self ):
result = jobDB.getJobJDL( str( self.__jobID ) , True )
classAdJob = ClassAd( result['Value'] )
executableFile = ""
if classAdJob.lookupAttribute( 'Executable' ):
executableFile = classAdJob.getAttributeString( 'Executable' )
tempPath = self.__tmpSandBoxDir
dirac = Dirac()
if not os.path.exists( tempPath ):
os.makedirs( tempPath )
settingJobSandBoxDir = dirac.getInputSandbox( self.__jobID, tempPath )
self.log.info( 'Writting temporal SandboxDir in Server', settingJobSandBoxDir )
moveData = self.__tmpSandBoxDir + "/InputSandbox" + str( self.__jobID )
HiveV1Cli = HiveV1Client( self.__User , self.__publicIP )
returned = HiveV1Cli.dataCopy( moveData, self.__tmpSandBoxDir )
self.log.info( 'Copy the job contain to the Hadoop Master with HIVE: ', returned )
jobInfo = jobDB.getJobAttributes( self.__jobID )
if not jobInfo['OK']:
return S_ERROR( jobInfo['Value'] )
proxy = ""
jobInfo = jobInfo['Value']
if gProxyManager.userHasProxy( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] ):
proxy = gProxyManager.downloadProxyToFile( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] )
else:
proxy = self.__requestProxyFromProxyManager( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] )
HiveJob = "InputSandbox" + str( self.__jobID ) + "/" + executableFile
HiveJobOutput = str( self.__jobID ) + "_" + executableFile + "_out"
returned = HiveV1Cli.jobSubmit( tempPath, HiveJob, proxy['chain'], HiveJobOutput )
self.log.info( 'Launch Hadoop-Hive job to the Master: ', returned )
if not returned['OK']:
return S_ERROR( returned['Message'] )
else:
self.log.info( 'Hadoop-Hive Job ID: ', returned['Value'] )
return S_OK( returned['Value'] )
def _getPilotProxyFromDIRACGroup( self, ownerDN, ownerGroup, requiredTimeLeft ):
"""
Download a limited pilot proxy with VOMS extensions depending on the group
"""
if gProxyManager.userHasProxy( ownerDN, ownerGroup ):
return gProxyManager.downloadProxyToFile( ownerDN, ownerGroup )
#Assign VOMS attribute
vomsAttr = CS.getVOMSAttributeForGroup( ownerGroup )
if not vomsAttr:
self.log.info( "Downloading a proxy without VOMS extensions for %s@%s" % ( ownerDN, ownerGroup ) )
return gProxyManager.downloadProxy( ownerDN, ownerGroup, limited = True,
requiredTimeLeft = requiredTimeLeft )
else:
self.log.info( "Downloading a proxy with '%s' VOMS extension for %s@%s" % ( vomsAttr, ownerDN, ownerGroup ) )
return gProxyManager.downloadVOMSProxy( ownerDN,
ownerGroup,
limited = True,
requiredTimeLeft = requiredTimeLeft,
requiredVOMSAttribute = vomsAttr )
def getShifterProxy(shifterType, fileName=False):
"""
This method returns a shifter's proxy
- shifterType : ProductionManager / DataManager...
"""
if fileName:
try:
os.makedirs(os.path.dirname(fileName))
except OSError:
pass
opsHelper = Operations()
userName = opsHelper.getValue(cfgPath("Shifter", shifterType, "User"), "")
if not userName:
return S_ERROR("No shifter User defined for %s" % shifterType)
result = CS.getDNForUsername(userName)
if not result["OK"]:
return result
userDN = result["Value"][0]
result = CS.findDefaultGroupForDN(userDN)
if not result["OK"]:
return result
defaultGroup = result["Value"]
userGroup = opsHelper.getValue(cfgPath("Shifter", shifterType, "Group"), defaultGroup)
vomsAttr = CS.getVOMSAttributeForGroup(userGroup)
if vomsAttr:
gLogger.info("Getting VOMS [%s] proxy for shifter %s@%s (%s)" % (vomsAttr, userName, userGroup, userDN))
result = gProxyManager.downloadVOMSProxyToFile(
userDN, userGroup, filePath=fileName, requiredTimeLeft=1200, cacheTime=4 * 43200
)
else:
gLogger.info("Getting proxy for shifter %s@%s (%s)" % (userName, userGroup, userDN))
result = gProxyManager.downloadProxyToFile(
userDN, userGroup, filePath=fileName, requiredTimeLeft=1200, cacheTime=4 * 43200
)
if not result["OK"]:
return result
chain = result["chain"]
fileName = result["Value"]
return S_OK({"DN": userDN, "username": userName, "group": userGroup, "chain": chain, "proxyFile": fileName})
def __prepareSecurityDetails( self, vomsFlag = True ):
""" Obtains the connection details for the client """
try:
credDict = self.getRemoteCredentials()
clientDN = credDict[ 'DN' ]
clientUsername = credDict['username']
clientGroup = credDict['group']
gLogger.debug( "Getting proxy for %s@%s (%s)" % ( clientUsername, clientGroup, clientDN ) )
if vomsFlag:
result = gProxyManager.downloadVOMSProxyToFile( clientDN, clientGroup )
else:
result = gProxyManager.downloadProxyToFile( clientDN, clientGroup )
if not result['OK']:
return result
gLogger.debug( "Updating environment." )
os.environ['X509_USER_PROXY'] = result['Value']
return result
except Exception, error:
exStr = "__getConnectionDetails: Failed to get client connection details."
gLogger.exception( exStr, '', error )
return S_ERROR( exStr )
def export_execProductionScript(self, script, workflow):
creds = self.__clientCredentials()
if creds['Group'] != 'lhcb_prmgr':
return S_ERROR("You have to be production manager")
result = gProxyManager.downloadProxyToFile(creds['DN'], creds['Group'],
filePath=False,
requiredTimeLeft=86400,
cacheTime=86400)
if not result['OK']:
return result
proxyFile = result['Value']
try:
f = tempfile.mkstemp()
os.write(f[0], workflow)
os.close(f[0])
fs = tempfile.mkstemp()
os.write(fs[0], script)
os.close(fs[0])
except OSError as msg:
gLogger.error("In temporary files creation: " + str(msg))
os.remove(proxyFile)
return S_ERROR(str(msg))
setenv = "source /opt/dirac/bashrc"
proxy = "X509_USER_PROXY=%s" % proxyFile
cmd = "python %s %s" % (fs[1], f[1])
try:
res = shellCall(1800, ["/bin/bash -c '%s;%s %s'" % (setenv, proxy, cmd)])
if res['OK']:
result = S_OK(str(res['Value'][1]) + str(res['Value'][2]))
else:
gLogger.error(res['Message'])
result = res
except Exception as msg: # pylint: disable=broad-except
gLogger.error("During execution: " + str(msg))
result = S_ERROR("Failed to execute: %s" % str(msg))
os.remove(f[1])
os.remove(fs[1])
os.remove(proxyFile)
return result
def getProxy(userDNs, userGroup, vomsAttr, proxyFilePath):
""" do the actual download of the proxy, trying the different DNs
"""
for userDN in userDNs:
if vomsAttr:
result = gProxyManager.downloadVOMSProxyToFile( userDN, userGroup,
requiredVOMSAttribute = vomsAttr,
filePath = proxyFilePath,
requiredTimeLeft = 3600,
cacheTime = 3600 )
else:
result = gProxyManager.downloadProxyToFile( userDN, userGroup,
filePath = proxyFilePath,
requiredTimeLeft = 3600,
cacheTime = 3600 )
if not result['OK']:
gLogger.warn( "Can't download proxy of '%s' to file" %userDN, result['Message'] )
else:
return result
return S_ERROR("Can't download proxy")
def submitNewBigPilot(self):
tempPath = self.__tmpSandBoxDir + str(self.__jobID)
dirac = Dirac()
if not os.path.exists(tempPath):
os.makedirs(tempPath)
settingJobSandBoxDir = dirac.getInputSandbox(self.__jobID, tempPath)
self.log.info("Writting temporal SandboxDir in Server", settingJobSandBoxDir)
jobXMLName = "job:" + str(self.__jobID) + ".xml"
with open(os.path.join(tempPath, jobXMLName), "wb") as temp_file:
temp_file.write(self.jobWrapper())
self.log.info("Writting temporal Hadoop Job.xml")
HadoopV1cli = HadoopV1Client(self.__User, self.__publicIP, self.__Port)
# returned = HadoopV1cli.dataCopy( tempPath, self.__tmpSandBoxDir )
# self.log.info( 'Copy the job contain to the Hadoop Master: ', returned )
jobInfo = jobDB.getJobAttributes(self.__jobID)
if not jobInfo["OK"]:
return S_ERROR(jobInfo["Value"])
proxy = ""
jobInfo = jobInfo["Value"]
if gProxyManager.userHasProxy(jobInfo["OwnerDN"], jobInfo["OwnerGroup"]):
proxy = gProxyManager.downloadProxyToFile(jobInfo["OwnerDN"], jobInfo["OwnerGroup"])
else:
proxy = self.__requestProxyFromProxyManager(jobInfo["OwnerDN"], jobInfo["OwnerGroup"])
returned = HadoopV1cli.submitPilotJob(tempPath, jobXMLName, proxy["chain"])
self.log.info("Launch Hadoop pilot to the Hadoop Master: ", returned)
if not returned["OK"]:
return S_ERROR(returned["Value"])
else:
self.log.info("Hadoop Job ID: ", returned["Value"])
return S_OK(returned["Value"])
def submitNewBigJob( self ):
tempPath = self.__tmpSandBoxDir + str( self.__jobID )
dirac = Dirac()
if not os.path.exists( tempPath ):
os.makedirs( tempPath )
settingJobSandBoxDir = dirac.getInputSandbox( self.__jobID, tempPath )
self.log.info( 'Writting temporal SandboxDir in Server', settingJobSandBoxDir )
jobXMLName = "job:" + str( self.__jobID ) + '.xml'
with open( os.path.join( tempPath, jobXMLName ), 'wb' ) as temp_file:
temp_file.write( self.jobWrapper() )
self.log.info( 'Writting temporal Hadoop Job.xml' )
HadoopV1cli = HadoopV2Client( self.__User , self.__publicIP )
returned = HadoopV1cli.dataCopy( tempPath, self.__tmpSandBoxDir )
self.log.info( 'Copy the job contain to the Hadoop Master: ', returned )
jobInfo = jobDB.getJobAttributes( self.__jobID )
if not jobInfo['OK']:
return S_ERROR( jobInfo['Value'] )
proxy = ""
jobInfo = jobInfo['Value']
if gProxyManager.userHasProxy( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] ):
proxy = gProxyManager.downloadProxyToFile( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] )
else:
proxy = self.__requestProxyFromProxyManager( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] )
returned = HadoopV1cli.jobSubmit( tempPath, jobXMLName, proxy['chain'] )
self.log.info( 'Launch Hadoop job to the Hadoop Master: ', returned )
if not returned['OK']:
return S_ERROR( returned['Message'] )
else:
self.log.info( 'Hadoop Job ID: ', returned['Value'] )
return S_OK( returned['Value'] )
def getProxy( userDNs, userGroup, vomsAttr, proxyFilePath ):
""" do the actual download of the proxy, trying the different DNs
"""
for userDN in userDNs:
if vomsAttr:
result = gProxyManager.downloadVOMSProxyToFile( userDN, userGroup,
requiredVOMSAttribute = vomsAttr,
filePath = proxyFilePath,
requiredTimeLeft = 3600,
cacheTime = 3600 )
else:
result = gProxyManager.downloadProxyToFile( userDN, userGroup,
filePath = proxyFilePath,
requiredTimeLeft = 3600,
cacheTime = 3600 )
if not result['OK']:
gLogger.error( "Can't download %sproxy " % ( 'VOMS' if vomsAttr else '' ),
"of '%s', group %s to file: " % ( userDN, userGroup ) + result['Message'] )
else:
return result
# If proxy not found for any DN, return an error
return S_ERROR( "Can't download proxy" )
"PIPELINE_PROCESSINSTANCE", "PIPELINE_ERRORADDRESS",
"JOBCONTROL_LOGFILE"
]
res = [i for i in required_variables if i not in pipeline_dict]
if len(res) != 0:
gLogger.error("Could not find critical variables for submission:",
str(res))
dexit(1)
op = Operations("glast.org")
#TODO: replace glast.org with VO-agnostic statement
shifter = op.getValue(
"Pipeline/Shifter",
"/DC=org/DC=doegrids/OU=People/CN=Stephan Zimmer 799865")
shifter_group = op.getValue("Pipeline/ShifterGroup", "glast_user")
result = gProxyManager.downloadProxyToFile(shifter,
shifter_group,
requiredTimeLeft=10000)
if not result['OK']:
gLogger.error("No valid proxy found; ", result['Message'])
dexit(1)
proxy = result['Value']
os.environ['X509_USER_PROXY'] = proxy
gLogger.info("using proxy %s" % proxy)
j = Job(stdout="DIRAC_wrapper.txt",
stderr="DIRAC_wrapper.txt") # specifies the logfile
input_sandbox_files = []
output_sandbox_files = ["*.log", "*.txt", "jobmeta.inf"]
if pipeline:
if opts.mailDebug:
Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
'Usage:',
' %s [option|cfgfile] ... JobID ...' % Script.scriptName,
'Arguments:',
' JobID: DIRAC Job ID' ] ) )
args = Script.getPositionalArgs()
from DIRAC import gLogger, exit as dexit
from DIRAC.Interfaces.API.Dirac import Dirac
from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
# get necessary credentials
op = Operations("glast.org")
shifter = op.getValue("Pipeline/Shifter","/DC=org/DC=doegrids/OU=People/CN=Stephan Zimmer 799865")
shifter_group = op.getValue("Pipeline/ShifterGroup","glast_user")
result = gProxyManager.downloadProxyToFile(shifter,shifter_group,requiredTimeLeft=10000)
if not result['OK']:
gLogger.error("ERROR: No valid proxy found; ",result['Message'])
dexit(1)
proxy = result[ 'Value' ]
environ['X509_USER_PROXY'] = proxy
gLogger.info("using proxy %s"%proxy)
dirac = Dirac(True,"myRepo.rep")
exitCode = 0
errorList = []
if len( args ) < 1:
Script.showHelp()
for job in args:
result = dirac.kill( job )
if result['OK']:
gLogger.info('Killed job %s' % ( job ))
Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
'Usage:',
' %s [option|cfgFile] UserName Role' % Script.scriptName,
'Arguments:',
' UserName: User DN'] ) )
Script.registerSwitch( 'R:', 'role=', "set the User DN." )
Script.parseCommandLine()
for unprocSw in Script.getUnprocessedSwitches():
if unprocSw[0] in ( "R", "role" ):
role = unprocSw[1]
args = Script.getPositionalArgs()
dn = ' '.join( args )
import os
uid = os.getuid()
from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager
res = gProxyManager.downloadProxyToFile( dn, role,
limited = False, requiredTimeLeft = 1200,
cacheTime = 43200, filePath = '/tmp/x509up_u%s' % uid, proxyToConnect = False,
token = False )
if not res['OK']:
print "Error downloading proxy", res['Message']
exit( 1 )
' %s [options] UserName Role' % Script.scriptName, 'Arguments:',
' UserName: User DN'
]))
Script.registerSwitch('R:', 'role=', "set the User DN.")
Script.parseCommandLine()
for unprocSw in Script.getUnprocessedSwitches():
if unprocSw[0] in ("R", "role"):
role = unprocSw[1]
args = Script.getPositionalArgs()
dn = ' '.join(args)
uid = os.getuid()
from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager
print("Getting proxy for User DN: %s, User role %s" % (dn, role))
res = gProxyManager.downloadProxyToFile(dn,
role,
limited=False,
requiredTimeLeft=1200,
cacheTime=43200,
filePath='/tmp/x509up_u%s' % uid,
proxyToConnect=False,
token=False)
if not res['OK']:
print("Error downloading proxy", res['Message'])
exit(1)
def submitNewBigJob( self ):
#1.- Creamos carpeta temporal
self.log.debug( 'Step1::: mkdir temp folder' )
tempPath = self.__tmpSandBoxDir + str( self.__jobID ) + "/"
dirac = Dirac()
if not os.path.exists( tempPath ):
os.makedirs( tempPath )
#2.- Introducimos el contenido del inputsandbox en la carpeta temporal
self.log.debug( 'Step2::: download inputsand to temp folder' )
settingJobSandBoxDir = dirac.getInputSandbox( self.__jobID, tempPath )
self.log.info( 'Writting temporal SandboxDir in Server', settingJobSandBoxDir )
moveData = tempPath + "/InputSandbox" + str( self.__jobID )
#3.- Move the data to client
self.log.debug( 'Step2::: download inputsandbox to temp folder' )
HadoopV1InteractiveCli = HadoopV1InteractiveClient( self.__User , self.__publicIP, self.__Port )
returned = HadoopV1InteractiveCli.dataCopy( tempPath, self.__tmpSandBoxDir )
self.log.debug( 'Returned of copy the job contain to the Hadoop Master with HadoopInteractive::: ', returned )
#3.- Get executable file
result = jobDB.getJobJDL( str( self.__jobID ) , True )
classAdJob = ClassAd( result['Value'] )
executableFile = ""
if classAdJob.lookupAttribute( 'Executable' ):
executableFile = classAdJob.getAttributeString( 'Executable' )
self.log.debug( 'Step3::: Get executable file: ', executableFile )
jobInfo = jobDB.getJobAttributes( self.__jobID )
if not jobInfo['OK']:
return S_ERROR( jobInfo['Value'] )
proxy = ""
jobInfo = jobInfo['Value']
if gProxyManager.userHasProxy( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] ):
proxy = gProxyManager.downloadProxyToFile( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] )
else:
proxy = self.__requestProxyFromProxyManager( jobInfo["OwnerDN"], jobInfo["OwnerGroup"] )
HadoopInteractiveJob = "InputSandbox" + str( self.__jobID ) + "/" + executableFile
HadoopInteractiveJobCommand = "InputSandbox" + str( self.__jobID ) + "/" + executableFile + " " + self.__JobName
HadoopInteractiveJobOutput = tempPath + str( self.__jobID ) + "_" + executableFile + "_out"
#4.- Creating second part of the job name
if ( len( re.split( " ", self.__JobName ) ) > 1 ):
#(name for random writter with -D)name_job = re.split( " ", self.__JobName )[0] + " " + re.split( " ", self.__JobName )[1] + " " + re.split( " ", self.__JobName )[2]
name_job = re.split( " ", self.__JobName )[0] + " " + re.split( " ", self.__JobName )[1]
#(name for random writter with -D)output_job = moveData + "/" + re.split( " ", self.__JobName )[3]
#(name for random writter with -D)cfg_job = ""
#(name for random writter with -D)if ( len( re.split( " ", self.__JobName ) ) > 4 ):
#(name for random writter with -D) cfg_job = moveData + "/" + re.split( " ", self.__JobName )[4]
#5.- Parsing execution command
#cmd = "hadoop jar " + tempPath + HadoopInteractiveJob + " " + name_job + " " + output_job + " " + cfg_job
cmd = "hadoop jar " + tempPath + HadoopInteractiveJob + " " + name_job + " " + tempPath + "/InputSandbox" + str( self.__jobID ) + "/" + "/dataset-USC-a-grep '[and]+'"
else:
dataset = re.split( "/", self.__Dataset )
count = 0
datasetname = ""
for dir in dataset:
count = count + 1
if ( count > 2 ):
datasetname = datasetname + "/" + dir
cmd = "hadoop jar " + tempPath + HadoopInteractiveJob + " " + self.__JobName + " " + datasetname + " " + tempPath + "/" + self.__JobName.replace( " ", "" ) + "_" + str( self.__jobID )
self.log.debug( 'Step4::: Making CMD for submission: ', cmd )
self.log.debug( 'Step5::: Submit file to hadoop: ' )
returned = HadoopV1InteractiveCli.jobSubmit( tempPath, HadoopInteractiveJob, proxy['chain'],
HadoopInteractiveJobOutput, cmd )
self.log.info( 'Launch Hadoop-HadoopInteractive job to the Master: ', returned )
if not returned['OK']:
return S_ERROR( returned['Message'] )
else:
self.log.info( 'Hadoop-HadoopInteractive Job ID: ', returned['Value'] )
return S_OK( returned['Value'] )
def wrapped_fcn( *args, **kwargs ):
userName = kwargs.pop( 'proxyUserName', '' )
userDN = kwargs.pop( 'proxyUserDN', '' )
userGroup = kwargs.pop( 'proxyUserGroup', '' )
vomsFlag = kwargs.pop( 'proxyWithVOMS', True )
proxyFilePath = kwargs.pop( 'proxyFilePath', False )
if ( userName or userDN ) and userGroup:
# Setup user proxy
originalUserProxy = os.environ.get( 'X509_USER_PROXY' )
if not userDN:
result = getDNForUsername( userName )
if not result[ 'OK' ]:
return result
userDN = result[ 'Value' ][0]
vomsAttr = ''
if vomsFlag:
vomsAttr = getVOMSAttributeForGroup( userGroup )
if vomsAttr:
result = gProxyManager.downloadVOMSProxyToFile( userDN, userGroup,
requiredVOMSAttribute = vomsAttr,
filePath = proxyFilePath,
requiredTimeLeft = 3600,
cacheTime = 3600 )
else:
result = gProxyManager.downloadProxyToFile( userDN, userGroup,
filePath = proxyFilePath,
requiredTimeLeft = 3600,
cacheTime = 3600 )
if not result['OK']:
gLogger.warn( "Can't download proxy to file", result['Message'] )
return result
proxyFile = result['Value']
os.environ['X509_USER_PROXY'] = proxyFile
# Check if the caller is executing with the host certificate
useServerCertificate = gConfig.useServerCertificate()
if useServerCertificate:
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false' )
try:
return fcn( *args, **kwargs )
except Exception as lException:
value = ','.join( [str( arg ) for arg in lException.args] )
exceptType = lException.__class__.__name__
return S_ERROR( "Exception - %s: %s" % ( exceptType, value ) )
finally:
# Restore the default host certificate usage if necessary
if useServerCertificate:
gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true' )
if originalUserProxy:
os.environ['X509_USER_PROXY'] = originalUserProxy
else:
os.environ.pop( 'X509_USER_PROXY' )
else:
# No proxy substitution requested
return fcn( *args, **kwargs )