def uploadProxyToMyProxy(params, DNAsUsername):
myProxy = MyProxy()
if DNAsUsername:
params.debugMsg("Uploading pilot proxy with group %s to %s..." %
(params.getDIRACGroup(), myProxy.getMyProxyServer()))
else:
params.debugMsg("Uploading user proxy with group %s to %s..." %
(params.getDIRACGroup(), myProxy.getMyProxyServer()))
retVal = myProxy.getInfo(proxyInfo['path'], useDNAsUserName=DNAsUsername)
if retVal['OK']:
remainingSecs = (int(params.getProxyRemainingSecs() / 3600) *
3600) - 7200
myProxyInfo = retVal['Value']
if 'timeLeft' in myProxyInfo and remainingSecs < myProxyInfo[
'timeLeft']:
params.debugMsg(" Already uploaded")
return True
retVal = generateProxy(params)
if not retVal['OK']:
print " There was a problem generating proxy to be uploaded to myproxy: %s" % retVal[
'Message']
return False
retVal = getProxyInfo(retVal['Value'])
if not retVal['OK']:
print " There was a problem generating proxy to be uploaded to myproxy: %s" % retVal[
'Message']
return False
generatedProxyInfo = retVal['Value']
retVal = myProxy.uploadProxy(generatedProxyInfo['path'],
useDNAsUserName=DNAsUsername)
if not retVal['OK']:
print " Can't upload to myproxy: %s" % retVal['Message']
return False
params.debugMsg(" Uploaded")
return True
def uploadProxyToMyProxy( params, DNAsUsername ):
""" Upload proxy to the MyProxy server
"""
myProxy = MyProxy()
if DNAsUsername:
gLogger.verbose( "Uploading pilot proxy with group %s to %s..." % ( params.getDIRACGroup(), myProxy.getMyProxyServer() ) )
else:
gLogger.verbose( "Uploading user proxy with group %s to %s..." % ( params.getDIRACGroup(), myProxy.getMyProxyServer() ) )
retVal = myProxy.getInfo( proxyInfo[ 'path' ], useDNAsUserName = DNAsUsername )
if retVal[ 'OK' ]:
remainingSecs = ( int( params.getProxyRemainingSecs() / 3600 ) * 3600 ) - 7200
myProxyInfo = retVal[ 'Value' ]
if 'timeLeft' in myProxyInfo and remainingSecs < myProxyInfo[ 'timeLeft' ]:
gLogger.verbose( " Already uploaded" )
return True
retVal = generateProxy( params )
if not retVal[ 'OK' ]:
print " There was a problem generating proxy to be uploaded to myproxy: %s" % retVal[ 'Message' ]
return False
retVal = getProxyInfo( retVal[ 'Value' ] )
if not retVal[ 'OK' ]:
print " There was a problem generating proxy to be uploaded to myproxy: %s" % retVal[ 'Message' ]
return False
generatedProxyInfo = retVal[ 'Value' ]
retVal = myProxy.uploadProxy( generatedProxyInfo[ 'path' ], useDNAsUserName = DNAsUsername )
if not retVal[ 'OK' ]:
print " Can't upload to myproxy: %s" % retVal[ 'Message' ]
return False
gLogger.verbose( " Uploaded" )
return True
def __setupProxy( self, ownerDN, ownerGroup ):
"""
Retrieve a proxy for the execution of the job
"""
if gConfig.getValue( '/DIRAC/Security/UseServerCertificate' , False ):
proxyResult = self.__requestProxyFromProxyManager( ownerDN, ownerGroup )
if not proxyResult['OK']:
self.log.error( 'Invalid Proxy', proxyResult['Message'] )
return S_ERROR( 'Failed to setup proxy: %s' % proxyResult[ 'Message' ] )
return S_OK( proxyResult['Value'] )
else:
ret = getProxyInfo( disableVOMS = True )
if not ret['OK']:
self.log.error( 'Invalid Proxy', ret['Message'] )
return S_ERROR( 'Invalid Proxy' )
proxyChain = ret['Value']['chain']
if not 'groupProperties' in ret['Value']:
print ret['Value']
print proxyChain.dumpAllToString()
self.log.error( 'Invalid Proxy', 'Group has no properties defined' )
return S_ERROR( 'Proxy has no group properties defined' )
if Properties.GENERIC_PILOT in ret['Value']['groupProperties']:
proxyResult = self.__requestProxyFromProxyManager( ownerDN, ownerGroup )
if not proxyResult['OK']:
self.log.error( 'Invalid Proxy', proxyResult['Message'] )
return S_ERROR( 'Failed to setup proxy: %s' % proxyResult[ 'Message' ] )
proxyChain = proxyResult['Value']
return S_OK( proxyChain )
def initialize(self, request):
""" Set default values to attributes,parameters
"""
if type(request) == types.NoneType:
# Set some defaults
for name in self.attributeNames:
self.attributes[name] = "Unknown"
status, self.attributes["RequestID"] = commands.getstatusoutput("uuidgen")
self.attributes["CreationTime"] = str(datetime.datetime.utcnow())
self.attributes["Status"] = "New"
result = getProxyInfo()
if result["OK"]:
proxyDict = result["Value"]
self.attributes["OwnerDN"] = proxyDict["identity"]
if "group" in proxyDict:
self.attributes["OwnerGroup"] = proxyDict["group"]
self.attributes["DIRACSetup"] = gConfig.getValue("/DIRAC/Setup", "LHCb-Development")
elif type(request) == types.InstanceType:
for attr in self.attributeNames:
self.attributes[attr] = request.attributes[attr]
# initialize request from an XML string
if type(request) in types.StringTypes:
for name in self.attributeNames:
self.attributes[name] = "Unknown"
self.parseRequest(request)
# Initialize request from another request
elif type(request) == types.InstanceType:
self.subrequests = copy.deepcopy(request.subrequests)
def initialize( self, request ):
""" Set default values to attributes,parameters
"""
if type( request ) == types.NoneType:
# Set some defaults
for name in self.attributeNames:
self.attributes[name] = 'Unknown'
self.attributes['CreationTime'] = str( Time.dateTime() )
self.attributes['Status'] = "New"
result = getProxyInfo()
if result['OK']:
proxyDict = result[ 'Value' ]
self.attributes['OwnerDN'] = proxyDict[ 'identity' ]
if 'group' in proxyDict:
self.attributes['OwnerGroup'] = proxyDict[ 'group' ]
self.attributes['DIRACSetup'] = gConfig.getValue( '/DIRAC/Setup', 'Unknown' )
elif type( request ) == types.InstanceType:
for attr in self.attributeNames:
self.attributes[attr] = request.attributes[attr]
# initialize request from an XML string
if type( request ) in types.StringTypes:
for name in self.attributeNames:
self.attributes[name] = 'Unknown'
self.parseRequest( request )
# Initialize request from another request
elif type( request ) == types.InstanceType:
self.subRequests = copy.deepcopy( request.subrequests )
def prepareTransformationTasks(self,transBody,taskDict,owner='',ownerGroup=''):
if (not owner) or (not ownerGroup):
res = getProxyInfo(False,False)
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
oJob = Job(transBody)
for taskNumber in sortList(taskDict.keys()):
paramsDict = taskDict[taskNumber]
transID = paramsDict['TransformationID']
self.log.verbose('Setting job owner:group to %s:%s' % (owner,ownerGroup))
oJob.setOwner(owner)
oJob.setOwnerGroup(ownerGroup)
transGroup = str(transID).zfill(8)
self.log.verbose('Adding default transformation group of %s' % (transGroup))
oJob.setJobGroup(transGroup)
constructedName = str(transID).zfill(8)+'_'+str(taskNumber).zfill(8)
self.log.verbose('Setting task name to %s' % constructedName)
oJob.setName(constructedName)
oJob._setParamValue('PRODUCTION_ID',str(transID).zfill(8))
oJob._setParamValue('JOB_ID',str(taskNumber).zfill(8))
inputData = None
for paramName,paramValue in paramsDict.items():
self.log.verbose('TransID: %s, TaskID: %s, ParamName: %s, ParamValue: %s' %(transID,taskNumber,paramName,paramValue))
if paramName=='InputData':
if paramValue:
self.log.verbose('Setting input data to %s' %paramValue)
oJob.setInputData(paramValue)
elif paramName=='Site':
if paramValue:
self.log.verbose('Setting allocated site to: %s' %(paramValue))
oJob.setDestination(paramValue)
elif paramValue:
self.log.verbose('Setting %s to %s' % (paramName,paramValue))
oJob._addJDLParameter(paramName,paramValue)
hospitalTrans = [int(x) for x in gConfig.getValue("/Operations/Hospital/Transformations",[])]
if int(transID) in hospitalTrans:
hospitalSite = gConfig.getValue("/Operations/Hospital/HospitalSite",'DIRAC.JobDebugger.ch')
hospitalCEs = gConfig.getValue("/Operations/Hospital/HospitalCEs",[])
oJob.setType('Hospital')
oJob.setDestination(hospitalSite)
oJob.setInputDataPolicy('download',dataScheduling=False)
if hospitalCEs:
oJob._addJDLParameter('GridRequiredCEs',hospitalCEs)
taskDict[taskNumber]['TaskObject'] = ''
res = self.getOutputData({'Job':oJob._toXML(),'TransformationID':transID,'TaskID':taskNumber,'InputData':inputData})
if not res ['OK']:
self.log.error("Failed to generate output data",res['Message'])
continue
for name,output in res['Value'].items():
oJob._addJDLParameter(name,string.join(output,';'))
taskDict[taskNumber]['TaskObject'] = Job(oJob._toXML())
return S_OK(taskDict)
def loadCredentials(self):
retVal = getProxyInfo()
if retVal['OK']:
credDict = retVal['Value']
self.commiterId = "%s@%s - %s" % (
credDict['username'], credDict['group'],
Time.dateTime().strftime("%Y-%m-%d %H:%M:%S"))
return retVal
return retVal
def loadCredentials( self ):
retVal = getProxyInfo()
if retVal[ 'OK' ]:
credDict = retVal[ 'Value' ]
self.commiterId = "%s@%s - %s" % ( credDict[ 'username' ],
credDict[ 'group' ],
Time.dateTime().strftime( "%Y-%m-%d %H:%M:%S" ) )
return retVal
return retVal
def execute( self ):
res = getProxyInfo( disableVOMS = True )
if not res['OK']:
return res
self.proxyInfoDict = res['Value']
res = self.monitorStageRequests()
return res
def _getCurrentUser( self ):
res = getProxyInfo( False, False )
if not res['OK']:
return self._errorReport( 'No proxy found in local environment', res['Message'] )
proxyInfo = res['Value']
gLogger.debug( formatProxyInfoAsString( proxyInfo ) )
if not proxyInfo.has_key( 'group' ):
return self._errorReport( 'Proxy information does not contain the group', res['Message'] )
res = getDNForUsername( proxyInfo['username'] )
if not res['OK']:
return self._errorReport( 'Failed to get proxies for user', res['Message'] )
return S_OK( proxyInfo['username'] )
def submitJob(self,executableFile,proxy, dummy=None):
""" Method to submit job, should be overridden in sub-class.
"""
# FIXME: when not running on a WN this will not work
ret = getProxyInfo()
if not ret['OK']:
pilotProxy = None
else:
pilotProxy = ret['Value']['path']
print 'pilotProxy', pilotProxy
payloadEnv = dict( os.environ )
payloadProxy = ''
if proxy:
self.log.verbose('Setting up proxy for payload')
result = self.writeProxyToFile(proxy)
if not result['OK']:
return result
payloadProxy = result['Value']
# pilotProxy = os.environ['X509_USER_PROXY']
payloadEnv[ 'X509_USER_PROXY' ] = payloadProxy
self.log.verbose('Starting process for monitoring payload proxy')
gThreadScheduler.addPeriodicTask(self.proxyCheckPeriod,self.monitorProxy,taskArgs=(pilotProxy,payloadProxy),executions=0,elapsedTime=0)
if not os.access(executableFile, 5):
os.chmod(executableFile,0755)
cmd = os.path.abspath(executableFile)
self.log.verbose('CE submission command: %s' %(cmd))
result = systemCall(0,cmd,callbackFunction = self.sendOutput,env=payloadEnv)
if payloadProxy:
os.unlink(payloadProxy)
ret = S_OK()
if not result['OK']:
self.log.error('Fail to run InProcess',result['Message'])
elif result['Value'][0] < 0:
self.log.error('InProcess Job Execution Failed')
self.log.info('Exit status:',result['Value'][0])
return S_ERROR('InProcess Job Execution Failed')
elif result['Value'][0] > 0:
self.log.error('Fail in payload execution')
self.log.info('Exit status:',result['Value'][0])
ret['PayloadFailed'] = result['Value'][0]
else:
self.log.debug('InProcess CE result OK')
self.submittedJobs += 1
return ret
def execute( self ):
self.log.info( "Start Execution" )
result = getProxyInfo()
if not result[ 'OK' ]:
return result
infoDict = result[ 'Value' ]
self.log.info( formatProxyInfoAsString( infoDict ) )
self._lookForCE()
self._infoFromCE()
self.log.info( "End Execution" )
return S_OK()
def execute(self):
self.log.info("Start Execution")
result = getProxyInfo()
if not result['OK']:
return result
infoDict = result['Value']
self.log.info(formatProxyInfoAsString(infoDict))
self._lookForCE()
self._infoFromCE()
self.log.info("End Execution")
return S_OK()
def __init__( self, script = None, stdout = 'std.out', stderr = 'std.err' ):
"""Instantiates the Workflow object and some default parameters.
"""
self.log = gLogger
self.section = COMPONENT_NAME
self.dbg = False
if gConfig.getValue( self.section + '/LogLevel', 'DEBUG' ) == 'DEBUG':
self.dbg = True
#gConfig.getValue('Tier0SE-tape','SEName')
self.stepCount = 0
self.owner = 'NotSpecified'
self.name = 'Name'
self.type = 'User'
self.priority = 1
vo = ''
ret = getProxyInfo( disableVOMS = True )
if ret['OK'] and 'group' in ret['Value']:
vo = getVOForGroup( ret['Value']['group'] )
self.group = vo
self.site = 'ANY' #ANY
#self.setup = 'Development'
self.origin = 'DIRAC'
self.stdout = stdout
self.stderr = stderr
self.logLevel = 'info'
self.executable = '$DIRACROOT/scripts/dirac-jobexec' # to be clarified
self.addToInputSandbox = []
self.addToOutputSandbox = []
self.addToInputData = []
self.systemConfig = 'ANY'
self.reqParams = {'MaxCPUTime': 'other.NAME>=VALUE',
'MinCPUTime': 'other.NAME<=VALUE',
'Site': 'other.NAME=="VALUE"',
'Platform': 'other.NAME=="VALUE"',
#'BannedSites': '!Member(other.Site,BannedSites)', #doesn't work unfortunately
'BannedSites': 'other.Site!="VALUE"',
'SystemConfig': 'Member("VALUE",other.CompatiblePlatforms)'}
##Add member to handle Parametric jobs
self.parametric = {}
self.script = script
if not script:
self.workflow = Workflow()
self.__setJobDefaults()
else:
self.workflow = Workflow( script )
#Global error dictionary
self.errorDict = {}
def _getCurrentUser(self):
res = getProxyInfo(False, False)
if not res['OK']:
return self._errorReport('No proxy found in local environment',
res['Message'])
proxyInfo = res['Value']
gLogger.debug(formatProxyInfoAsString(proxyInfo))
if not proxyInfo.has_key('group'):
return self._errorReport(
'Proxy information does not contain the group', res['Message'])
res = getDNForUsername(proxyInfo['username'])
if not res['OK']:
return self._errorReport('Failed to get proxies for user',
res['Message'])
return S_OK(proxyInfo['username'])
def submitTasks(self):
gLogger.info("submitTasks: Submitting tasks for transformations")
res = getProxyInfo(False,False)
if not res['OK']:
gLogger.error("submitTasks: Failed to determine credentials for submission",res['Message'])
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
gLogger.info("submitTasks: Tasks will be submitted with the credentials %s:%s" % (owner,ownerGroup))
# Get the transformations which should be submitted
tasksPerLoop = self.am_getOption('TasksPerLoop',50)
status = self.am_getOption('SubmitStatus',['Active','Completing'])
res = self._selectTransformations(transType=self.transType,status=status)
if not res['OK']:
return res
for transformation in res['Value']:
transID = transformation['TransformationID']
transBody = transformation['Body']
res = self.transClient.getTasksToSubmit(transID,tasksPerLoop)
if not res['OK']:
gLogger.error("submitTasks: Failed to obtain tasks for transformation", "%s %s" % (transID,res['Message']))
continue
tasks = res['Value']['JobDictionary']
if not tasks:
gLogger.verbose("submitTasks: No tasks found for submission for transformation %s" % transID)
continue
gLogger.info("submitTasks: Obtained %d tasks for submission for transformation %s" % (len(tasks),transID))
res = self.prepareTransformationTasks(transBody,tasks,owner,ownerGroup)
if not res['OK']:
gLogger.error("submitTasks: Failed to prepare tasks for transformation", "%s %s" % (transID,res['Message']))
continue
res = self.submitTransformationTasks(res['Value'])
if not res['OK']:
gLogger.error("submitTasks: Failed to submit prepared tasks for transformation", "%s %s" % (transID,res['Message']))
continue
res = self.updateDBAfterTaskSubmission(res['Value'])
if not res['OK']:
gLogger.error("submitTasks: Failed to update DB after task submission for transformation", "%s %s" % (transID,res['Message']))
continue
gLogger.info("submitTasks: Submission of transformation tasks complete")
return S_OK()
def CheckAndGetProdProxy():
""" Check if current proxy is a production one, and if not call the L{getNewProxy} method.
"""
result = getProxyInfo()
if not result['OK']:
print 'Could not obtain proxy information: %s' % result['Message']
if getNewProxy():
return S_ERROR("dirac-proxy-init failed")
elif not result['Value'].has_key('group'):
print 'Could not get group from proxy'
getNewProxy()
if getNewProxy():
return S_ERROR("dirac-proxy-init failed")
group = result['Value']['group']
if not group == 'ilc_prod':
print 'You do not have a valid group'
getNewProxy()
if getNewProxy():
return S_ERROR("dirac-proxy-init failed")
return S_OK()
def do_id(self,args):
""" Get user identity
"""
result = getProxyInfo()
if not result['OK']:
print "Error: %s" % result['Message']
return
user = result['Value']['username']
group = result['Value']['group']
result = self.fc.getUsers()
if not result['OK']:
print "Error: %s" % result['Message']
return
userDict = result['Value']
result = self.fc.getGroups()
if not result['OK']:
print "Error: %s" % result['Message']
return
groupDict = result['Value']
idUser = userDict.get(user,0)
idGroup = groupDict.get(group,0)
print "user=%d(%s) group=%d(%s)" % (idUser,user,idGroup,group)
def executeGridCommand( proxy, cmd, gridEnvScript = None ):
""" Execute cmd tuple after sourcing GridEnv
"""
currentEnv = dict( os.environ )
if gridEnvScript:
ret = sourceEnv( 10, [gridEnvScript] )
if not ret['OK']:
return S_ERROR( 'Failed sourcing GridEnv: %s' % ret['Message'] )
gridEnv = ret['outputEnv']
#
# Preserve some current settings if they are there
#
if currentEnv.has_key( 'X509_VOMS_DIR' ):
gridEnv['X509_VOMS_DIR'] = currentEnv['X509_VOMS_DIR']
if currentEnv.has_key( 'X509_CERT_DIR' ):
gridEnv['X509_CERT_DIR'] = currentEnv['X509_CERT_DIR']
else:
gridEnv = currentEnv
if not proxy:
res = getProxyInfo()
if not res['OK']:
return res
gridEnv['X509_USER_PROXY' ] = res['Value']['path']
elif type( proxy ) in types.StringTypes:
if os.path.exists( proxy ):
gridEnv[ 'X509_USER_PROXY' ] = proxy
else:
return S_ERROR( 'Can not treat proxy passed as a string' )
else:
ret = gProxyManager.dumpProxyToFile( proxy )
if not ret['OK']:
return ret
gridEnv[ 'X509_USER_PROXY' ] = ret['Value']
return systemCall( 120, cmd, env = gridEnv )
def __setupProxy(self, ownerDN, ownerGroup):
"""
Retrieve a proxy for the execution of the job
"""
if gConfig.getValue('/DIRAC/Security/UseServerCertificate', False):
proxyResult = self.__requestProxyFromProxyManager(
ownerDN, ownerGroup)
if not proxyResult['OK']:
self.log.error('Invalid Proxy', proxyResult['Message'])
return S_ERROR('Failed to setup proxy: %s' %
proxyResult['Message'])
return S_OK(proxyResult['Value'])
else:
ret = getProxyInfo(disableVOMS=True)
if not ret['OK']:
self.log.error('Invalid Proxy', ret['Message'])
return S_ERROR('Invalid Proxy')
proxyChain = ret['Value']['chain']
if not 'groupProperties' in ret['Value']:
print ret['Value']
print proxyChain.dumpAllToString()
self.log.error('Invalid Proxy',
'Group has no properties defined')
return S_ERROR('Proxy has no group properties defined')
if Properties.GENERIC_PILOT in ret['Value']['groupProperties']:
proxyResult = self.__requestProxyFromProxyManager(
ownerDN, ownerGroup)
if not proxyResult['OK']:
self.log.error('Invalid Proxy', proxyResult['Message'])
return S_ERROR('Failed to setup proxy: %s' %
proxyResult['Message'])
proxyChain = proxyResult['Value']
return S_OK(proxyChain)
args = Script.getPositionalArgs()
if not args:
Script.showHelp()
from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient
from DIRAC.Core.Security.Misc import getProxyInfo
from DIRAC import gConfig, gLogger
from DIRAC.Core.DISET.RPCClient import RPCClient
from DIRAC.ResourceStatusSystem.Utilities.CS import getMailForUser
nc = NotificationClient()
s = RPCClient( "ResourceStatus/ResourceStatus" )
res = getProxyInfo()
if not res['OK']:
gLogger.error( "Failed to get proxy information", res['Message'] )
DIRAC.exit( 2 )
userName = res['Value']['username']
group = res['Value']['group']
if group not in ( 'diracAdmin', 'lhcb_prod' ):
gLogger.error( "You must be lhcb_prod or diracAdmin to execute this script" )
gLogger.info( "Please issue 'lhcb-proxy-init -g lhcb_prod' or 'lhcb-proxy-init -g diracAdmin'" )
DIRAC.exit( 2 )
for arg in args:
g = s.whatIs( arg )
res = s.reAssignToken( g, arg, userName )
if not res['OK']:
gLogger.error( "Problem with re-assigning token for %s: " % res['Message'] )
DIRAC.gLogger.debug('/DIRAC/Gateways/%s =' % DIRAC.siteName(),
gatewayServer)
Script.localCfg.addDefaultEntry('/DIRAC/Gateways/%s' % DIRAC.siteName(),
gatewayServer)
# Create the local dirac.cfg if it is not yet there
if not os.path.exists(DIRAC.gConfig.diracConfigFilePath):
configDir = os.path.dirname(DIRAC.gConfig.diracConfigFilePath)
if not os.path.exists(configDir):
os.makedirs(configDir)
update = True
DIRAC.gConfig.dumpLocalCFGToFile(DIRAC.gConfig.diracConfigFilePath)
# We need user proxy or server certificate to continue
if not useServerCert:
result = getProxyInfo()
if not result['OK']:
DIRAC.gLogger.notice('No user proxy available')
DIRAC.gLogger.notice(
'Create one using %s and execute again' %
os.path.join(DIRAC.rootPath, 'scripts', 'dirac-proxy-init'))
sys.exit(0)
else:
Script.enableCS()
else:
Script.localCfg.addDefaultEntry('/DIRAC/Security/UseServerCertificate',
'yes')
Script.enableCS()
if includeAllServers:
DIRAC.gConfig.setOptionValue('/DIRAC/Configuration/Servers',
if cTimeStruct < maxCTime:
return True
return False
verbose = False
if days or months or years:
verbose = True
totalDays = 0
if years:
totalDays += 365 * years
if months:
totalDays += 30 * months
if days:
totalDays += days
res = getProxyInfo( False, False )
if not res['OK']:
gLogger.error( "Failed to get client proxy information.", res['Message'] )
DIRAC.exit( 2 )
proxyInfo = res['Value']
username = proxyInfo['username']
userBase = '/%s/user/%s/%s' % ( vo, username[0], username )
if not baseDir:
baseDir = userBase
gLogger.info( 'Will search for files in %s' % baseDir )
activeDirs = [baseDir]
allFiles = []
emptyDirs = []
while len( activeDirs ) > 0:
def uploadProxyToDIRACProxyManager( params ):
""" Upload proxy to the DIRAC ProxyManager service
"""
gLogger.verbose( "Uploading user pilot proxy with group %s..." % ( params.getDIRACGroup() ) )
params.onTheFly = True
retVal = uploadProxy( params )
if not retVal[ 'OK' ]:
print " There was a problem generating proxy to be uploaded proxy manager: %s" % retVal[ 'Message' ]
return False
return True
Script.enableCS()
retVal = getProxyInfo( retVal[ 'Value' ] )
if not retVal[ 'OK' ]:
print "Can't create a proxy: %s" % retVal[ 'Message' ]
sys.exit( 1 )
proxyInfo = retVal[ 'Value' ]
if 'username' not in proxyInfo:
print "Not authorized in DIRAC"
sys.exit( 1 )
retVal = CS.getGroupsForUser( proxyInfo[ 'username' ] )
if not retVal[ 'OK' ]:
print "No groups defined for user %s" % proxyInfo[ 'username' ]
sys.exit( 1 )
availableGroups = retVal[ 'Value' ]
def uploadProxyToDIRACProxyManager(params):
params.debugMsg("Uploading user pilot proxy with group %s..." %
(params.getDIRACGroup()))
params.onTheFly = True
retVal = uploadProxy(params)
if not retVal['OK']:
print " There was a problem generating proxy to be uploaded proxy manager: %s" % retVal[
'Message']
return False
return True
Script.enableCS()
retVal = getProxyInfo(retVal['Value'])
if not retVal['OK']:
print "Can't create a proxy: %s" % retVal['Message']
sys.exit(1)
proxyInfo = retVal['Value']
if 'username' not in proxyInfo:
print "Not authorized in DIRAC"
sys.exit(1)
retVal = CS.getGroupsForUser(proxyInfo['username'])
if not retVal['OK']:
print "No groups defined for user %s" % proxyInfo['username']
sys.exit(1)
availableGroups = retVal['Value']
from DIRAC.Core.Base import Script
Script.setUsageMessage("""
Get the currently defined user data volume quotas
Usage:
%s [options]
""" % Script.scriptName)
Script.parseCommandLine( ignoreErrors = False )
import DIRAC
from DIRAC import gLogger, gConfig
from DIRAC.Core.Security.Misc import getProxyInfo
res = getProxyInfo( False, False )
if not res['OK']:
gLogger.error( "Failed to get client proxy information.", res['Message'] )
DIRAC.exit( 2 )
proxyInfo = res['Value']
username = proxyInfo['username']
try:
quota = gConfig.getValue('/Registry/DefaultStorageQuota', 0. )
quota = gConfig.getValue('/Registry/Users/%s/Quota' % username, quota )
gLogger.notice('Current quota found to be %.1f GB' % quota)
DIRAC.exit(0)
except Exception,x:
gLogger.exception("Failed to convert retrieved quota",'',x)
DIRAC.exit(-1)
def submitTasks(self):
gLogger.info("submitTasks: Submitting tasks for transformations")
res = getProxyInfo(False, False)
if not res['OK']:
gLogger.error(
"submitTasks: Failed to determine credentials for submission",
res['Message'])
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
gLogger.info(
"submitTasks: Tasks will be submitted with the credentials %s:%s" %
(owner, ownerGroup))
# Get the transformations which should be submitted
tasksPerLoop = self.am_getOption('TasksPerLoop', 50)
status = self.am_getOption('SubmitStatus', ['Active', 'Completing'])
res = self._selectTransformations(transType=self.transType,
status=status)
if not res['OK']:
return res
for transformation in res['Value']:
transID = transformation['TransformationID']
transBody = transformation['Body']
res = self.transClient.getTasksToSubmit(transID, tasksPerLoop)
if not res['OK']:
gLogger.error(
"submitTasks: Failed to obtain tasks for transformation",
"%s %s" % (transID, res['Message']))
continue
tasks = res['Value']['JobDictionary']
if not tasks:
gLogger.verbose(
"submitTasks: No tasks found for submission for transformation %s"
% transID)
continue
gLogger.info(
"submitTasks: Obtained %d tasks for submission for transformation %s"
% (len(tasks), transID))
res = self.prepareTransformationTasks(transBody, tasks, owner,
ownerGroup)
if not res['OK']:
gLogger.error(
"submitTasks: Failed to prepare tasks for transformation",
"%s %s" % (transID, res['Message']))
continue
res = self.submitTransformationTasks(res['Value'])
if not res['OK']:
gLogger.error(
"submitTasks: Failed to submit prepared tasks for transformation",
"%s %s" % (transID, res['Message']))
continue
res = self.updateDBAfterTaskSubmission(res['Value'])
if not res['OK']:
gLogger.error(
"submitTasks: Failed to update DB after task submission for transformation",
"%s %s" % (transID, res['Message']))
continue
gLogger.info(
"submitTasks: Submission of transformation tasks complete")
return S_OK()
for switch in Script.getUnprocessedSwitches():
if switch[0].lower() == "e" or switch[0].lower() == "extension":
hours = int( switch[1] )
from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient
from DIRAC.Core.Security.Misc import getProxyInfo
from DIRAC import gConfig, gLogger
from DIRAC.Core.DISET.RPCClient import RPCClient
from DIRAC.ResourceStatusSystem.Utilities.CS import getMailForUser
nc = NotificationClient()
s = RPCClient( "ResourceStatus/ResourceStatus" )
res = getProxyInfo()
if not res['OK']:
gLogger.error( "Failed to get proxy information", res['Message'] )
DIRAC.exit( 2 )
userName = res['Value']['username']
group = res['Value']['group']
if group not in ( 'diracAdmin', 'lhcb_prod' ):
gLogger.error( "You must be lhcb_prod or diracAdmin to execute this script" )
gLogger.info( "Please issue 'lhcb-proxy-init -g lhcb_prod' or 'lhcb-proxy-init -g diracAdmin'" )
DIRAC.exit( 2 )
for arg in args:
g = s.whatIs( arg )
res = s.extendToken( g, arg, hours )
if not res['OK']:
gLogger.error( "Problem with extending: %s" % res['Message'] )
' CE: Name of the CE(ie: ce111.cern.ch)'] ) )
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
from DIRAC.Core.Security.Misc import getProxyInfo
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup
if not len( args ) == 1:
Script.showHelp()
ceName = args[0]
host = None
vo = None
ret = getProxyInfo( disableVOMS = True )
if ret['OK'] and 'group' in ret['Value']:
vo = getVOForGroup( ret['Value']['group'] )
for unprocSw in Script.getUnprocessedSwitches():
if unprocSw[0] in ( "H", "host" ):
host = unprocSw[1]
if unprocSw[0] in ( "V", "vo" ):
vo = unprocSw[1]
if not vo:
Script.gLogger.error( 'Could not determine VO' )
Script.showHelp()
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
diracAdmin = DiracAdmin()
if gatewayServer:
DIRAC.gLogger.debug( '/DIRAC/Gateways/%s =' % DIRAC.siteName(), gatewayServer )
Script.localCfg.addDefaultEntry( '/DIRAC/Gateways/%s' % DIRAC.siteName(), gatewayServer )
# Create the local dirac.cfg if it is not yet there
if not os.path.exists( DIRAC.gConfig.diracConfigFilePath ):
configDir = os.path.dirname( DIRAC.gConfig.diracConfigFilePath )
if not os.path.exists( configDir ):
os.makedirs( configDir )
update = True
DIRAC.gConfig.dumpLocalCFGToFile( DIRAC.gConfig.diracConfigFilePath )
# We need user proxy or server certificate to continue
if not useServerCert:
result = getProxyInfo()
if not result['OK']:
DIRAC.gLogger.notice( 'No user proxy available' )
DIRAC.gLogger.notice( 'Create one using %s and execute again' % os.path.join( DIRAC.rootPath, 'scripts', 'dirac-proxy-init' ) )
sys.exit( 0 )
else:
Script.enableCS()
else:
Script.localCfg.addDefaultEntry( '/DIRAC/Security/UseServerCertificate', 'yes' )
Script.enableCS()
if includeAllServers:
DIRAC.gConfig.setOptionValue( '/DIRAC/Configuration/Servers', ','.join( DIRAC.gConfig.getServersList() ) )
DIRAC.gLogger.debug( '/DIRAC/Configuration/Servers =', ','.join( DIRAC.gConfig.getServersList() ) )
if update:
