def getInfo():
"""Retrieve information about setup, etc."""
records = []
records.append(('Setup', gConfig.getValue('/DIRAC/Setup', 'Unknown')))
records.append(('ConfigurationServer', gConfig.getValue('/DIRAC/Configuration/Servers', [])))
records.append(('Installation path', DIRAC.rootPath))
if os.path.exists(os.path.join(DIRAC.rootPath, DIRAC.getPlatform(), 'bin', 'mysql')):
records.append(('Installation type', 'server'))
else:
records.append(('Installation type', 'client'))
records.append(('Platform', DIRAC.getPlatform()))
ret = getProxyInfo(disableVOMS=False)
if ret['OK']:
print(pprint.pformat(ret))
if 'group' in ret['Value']:
vo = getVOForGroup(ret['Value']['group'])
else:
vo = getVOForGroup('')
if not vo:
vo = "None"
records.append(('VirtualOrganization', vo))
if 'identity' in ret['Value']:
records.append(('User DN', ret['Value']['identity']))
if 'secondsLeft' in ret['Value']:
records.append(('Proxy validity, secs', {'Value': str(ret['Value']['secondsLeft']), 'Just': 'L'}))
if gConfig.getValue('/DIRAC/Security/UseServerCertificate', True):
records.append(('Use Server Certificate', 'Yes'))
else:
records.append(('Use Server Certificate', 'No'))
if gConfig.getValue('/DIRAC/Security/SkipCAChecks', False):
records.append(('Skip CA Checks', 'Yes'))
else:
records.append(('Skip CA Checks', 'No'))
try:
import gfalthr # pylint: disable=import-error
records.append(('gfal version', gfalthr.gfal_version()))
except BaseException:
pass
fields = ['Option', 'Value']
return fields, records
def __getUserDetails( self ):
""" Get details on user account
"""
credentials = self.getRemoteCredentials()
if credentials:
diracUser = credentials.get( "username" )
diracGroup = credentials.get( "group" )
if not ( diracUser and diracGroup ):
return S_ERROR( 'Failed to get DIRAC user name and/or group' )
vo = getVOForGroup( diracGroup )
diracHome = ''
if vo:
diracHome = '/%s/user/%s/%s' % ( vo, diracUser[0], diracUser )
cfgPath = self.serviceInfoDict[ 'serviceSectionPath' ]
gLogger.debug( "cfgPath: %s" % cfgPath )
irodsUser = gConfig.getValue( "%s/UserCredentials/%s/iRodsUser" % ( cfgPath , diracUser ) , diracUser )
irodsHome = gConfig.getValue( "%s/UserCredentials/%s/iRodsHome" % ( cfgPath , diracUser ) , '' )
irodsGroup = gConfig.getValue( "%s/UserCredentials/%s/iRodsGroup" % ( cfgPath , diracUser ) , '' )
irodsPassword = gConfig.getValue( "%s/UserCredentials/%s/iRodsPassword" % ( cfgPath , diracUser ) , '' )
resultDict = {}
resultDict['DIRACUser'] = diracUser
resultDict['DIRACGroup'] = diracGroup
resultDict['DIRACHome'] = diracHome
resultDict['iRodsUser'] = irodsUser
resultDict['iRodsGroup'] = irodsGroup
resultDict['iRodsHome'] = irodsHome
resultDict['iRodsPassword'] = irodsPassword
return S_OK( resultDict )
def __init__(self, *args, **kwargs):
super(FileCatalogHandler, self).__init__(*args, **kwargs)
sessionData = self.getSessionData()
self.user = sessionData['user'].get('username', '')
self.group = sessionData['user'].get('group', '')
self.vo = getVOForGroup(self.group)
self.fc = FileCatalog(vo=self.vo)
def configureFromSection(self, mySection):
"""
reload from CS
"""
self.pilot = gConfig.getValue(mySection + "/PilotScript", self.pilot)
self.installVersion = gConfig.getValue(mySection + "/Version", self.installVersion)
self.extraPilotOptions = gConfig.getValue(mySection + "/ExtraPilotOptions", self.extraPilotOptions)
self.install = gConfig.getValue(mySection + "/InstallScript", self.install)
self.extraModules = gConfig.getValue(mySection + "/ExtraPilotModules", []) + self.extraModules
self.installProject = gConfig.getValue(mySection + "/Project", self.installProject)
self.installation = gConfig.getValue(mySection + "/Installation", self.installation)
self.maxJobsInFillMode = gConfig.getValue(mySection + "/MaxJobsInFillMode", self.maxJobsInFillMode)
self.targetGrids = gConfig.getValue(mySection + "/TargetGrids", self.targetGrids)
self.enableListMatch = gConfig.getValue(mySection + "/EnableListMatch", self.enableListMatch)
self.listMatchDelay = gConfig.getValue(mySection + "/ListMatchDelay", self.listMatchDelay)
self.errorClearTime = gConfig.getValue(mySection + "/ErrorClearTime", self.errorClearTime)
self.errorTicketTime = gConfig.getValue(mySection + "/ErrorTicketTime", self.errorTicketTime)
self.errorMailAddress = gConfig.getValue(mySection + "/ErrorMailAddress", self.errorMailAddress)
self.alarmMailAddress = gConfig.getValue(mySection + "/AlarmMailAddress", self.alarmMailAddress)
self.mailFromAddress = gConfig.getValue(mySection + "/MailFromAddress", self.mailFromAddress)
self.privatePilotFraction = gConfig.getValue(mySection + "/PrivatePilotFraction", self.privatePilotFraction)
virtualOrganization = gConfig.getValue(mySection + "/VirtualOrganization", "")
if not virtualOrganization:
virtualOrganization = getVOForGroup("NonExistingGroup")
if not virtualOrganization:
virtualOrganization = self.virtualOrganization
self.virtualOrganization = virtualOrganization
def web_getLaunchpadSetupWithLFNs(self):
""" Method obtain launchpad setup with pre-selected LFNs as input data parameter,
the caller js client will use setup to open an new Launchpad
"""
# On the fly file catalog for advanced launchpad
if not hasattr(self, 'fc'):
userData = self.getSessionData()
group = str(userData["user"]["group"])
vo = getVOForGroup(group)
self.fc = FileCatalog(vo=vo)
self.set_header('Content-type', 'text/plain')
arguments = self.request.arguments
gLogger.always("submit: incoming arguments %s to getLaunchpadSetupWithLFNs" % arguments)
lfnList = str(arguments['path'][0]).split(',')
# Modified for Eiscat
# Checks if the experiments folder in lfn list has a rtg_def.m file at some subfolder
gLogger.always("submit: checking if some rtg_def.m", arguments)
processed = []
metaDict = {'type': 'info'}
for lfn in lfnList:
pos_relative = lfn.find("/")
pos_relative = lfn.find("/", pos_relative + 1)
pos_relative = lfn.find("/", pos_relative + 1)
pos_relative = lfn.find("/", pos_relative + 1)
pos_relative = lfn.find("/", pos_relative + 1)
experiment_lfn = lfn[0:pos_relative]
if experiment_lfn in processed:
continue
processed.append(experiment_lfn)
gLogger.always( "checking rtg_def.m in %s" % experiment_lfn )
result = self.fc.findFilesByMetadata( metaDict, path=str(experiment_lfn) )
if not result['OK'] or not result['Value']:
gLogger.error( "Failed to get type info from $s, %s" % (experiment_lfn,result[ "Message" ]) )
continue
for candidate_lfn in result['Value']:
if candidate_lfn.find('rtg_def.m') > 0:
lfnList.append(candidate_lfn)
# End modified
ptlfn = ''
for lfn in lfnList:
ptlfn += (', ' + lfn) if ptlfn else lfn
params = self.defaultParams.copy()
params["InputData"] = [1, ptlfn]
obj = Operations(vo=vo)
predefinedSets = {}
launchpadSections = obj.getSections("Launchpad")
if launchpadSections['OK']:
for section in launchpadSections["Value"]:
predefinedSets[section] = {}
sectionOptions = obj.getOptionsDict("Launchpad/" + section)
pprint.pprint(sectionOptions)
if sectionOptions['OK']:
predefinedSets[section] = sectionOptions["Value"]
self.write({"success": "true", "result": params, "predefinedSets": predefinedSets})
def configureFromSection( self, mySection ):
"""
reload from CS
"""
self.pilot = gConfig.getValue( mySection + '/PilotScript' , self.pilot )
self.installVersion = gConfig.getValue( mySection + '/Version' , self.installVersion )
self.extraPilotOptions = gConfig.getValue( mySection + '/ExtraPilotOptions' , self.extraPilotOptions )
self.install = gConfig.getValue( mySection + '/InstallScript' , self.install )
self.extraModules = gConfig.getValue( mySection + '/ExtraPilotModules' , [] ) + self.extraModules
self.installProject = gConfig.getValue( mySection + '/Project' , self.installProject )
self.installation = gConfig.getValue( mySection + '/Installation' , self.installation )
self.maxJobsInFillMode = gConfig.getValue( mySection + '/MaxJobsInFillMode' , self.maxJobsInFillMode )
self.targetGrids = gConfig.getValue( mySection + '/TargetGrids' , self.targetGrids )
self.enableListMatch = gConfig.getValue( mySection + '/EnableListMatch' , self.enableListMatch )
self.listMatchDelay = gConfig.getValue( mySection + '/ListMatchDelay' , self.listMatchDelay )
self.errorClearTime = gConfig.getValue( mySection + '/ErrorClearTime' , self.errorClearTime )
self.errorTicketTime = gConfig.getValue( mySection + '/ErrorTicketTime' , self.errorTicketTime )
self.errorMailAddress = gConfig.getValue( mySection + '/ErrorMailAddress' , self.errorMailAddress )
self.alarmMailAddress = gConfig.getValue( mySection + '/AlarmMailAddress' , self.alarmMailAddress )
self.mailFromAddress = gConfig.getValue( mySection + '/MailFromAddress' , self.mailFromAddress )
self.privatePilotFraction = gConfig.getValue( mySection + '/PrivatePilotFraction' , self.privatePilotFraction )
virtualOrganization = gConfig.getValue( mySection + '/VirtualOrganization' , '' )
if not virtualOrganization:
virtualOrganization = getVOForGroup( 'NonExistingGroup' )
if not virtualOrganization:
virtualOrganization = self.virtualOrganization
self.virtualOrganization = virtualOrganization
def __init__(self, *args, **kwargs ): super( FileCatalogHandler, self ).__init__( *args, **kwargs ) sessionData = self.getSessionData() self.user = sessionData['user'].get( 'username', '' ) self.group = sessionData['user'].get( 'group', '' ) self.vo = getVOForGroup( self.group ) self.fc = FileCatalog( vo = self.vo )
def configureFromSection( self, mySection ):
"""
reload from CS
"""
self.pilot = gConfig.getValue( mySection + '/PilotScript' , self.pilot )
self.installVersion = gConfig.getValue( mySection + '/DIRACVersion' , self.installVersion )
self.extraPilotOptions = gConfig.getValue( mySection + '/ExtraPilotOptions' , self.extraPilotOptions )
self.install = gConfig.getValue( mySection + '/InstallScript' , self.install )
self.installInstallation = gConfig.getValue( mySection + '/Installation' , self.installInstallation )
self.maxJobsInFillMode = gConfig.getValue( mySection + '/MaxJobsInFillMode' , self.maxJobsInFillMode )
self.enableListMatch = gConfig.getValue( mySection + '/EnableListMatch' , self.enableListMatch )
self.listMatchDelay = gConfig.getValue( mySection + '/ListMatchDelay' , self.listMatchDelay )
self.errorClearTime = gConfig.getValue( mySection + '/ErrorClearTime' , self.errorClearTime )
self.errorTicketTime = gConfig.getValue( mySection + '/ErrorTicketTime' , self.errorTicketTime )
self.errorMailAddress = gConfig.getValue( mySection + '/ErrorMailAddress' , self.errorMailAddress )
self.alarmMailAddress = gConfig.getValue( mySection + '/AlarmMailAddress' , self.alarmMailAddress )
self.mailFromAddress = gConfig.getValue( mySection + '/MailFromAddress' , self.mailFromAddress )
self.genericPilotDN = gConfig.getValue( mySection + '/GenericPilotDN' , self.genericPilotDN )
self.genericPilotGroup = gConfig.getValue( mySection + '/GenericPilotGroup' , self.genericPilotGroup )
self.privatePilotFraction = gConfig.getValue( mySection + '/PrivatePilotFraction' , self.privatePilotFraction )
virtualOrganization = gConfig.getValue( mySection + '/VirtualOrganization' , '' )
if not virtualOrganization:
virtualOrganization = getVOForGroup( 'NonExistingGroup' )
if not virtualOrganization:
virtualOrganization = self.virtualOrganization
self.virtualOrganization = virtualOrganization
def __init__(self,
rpcClient=None,
transferClient=None,
smdb=False,
**kwargs):
""" Constructor
:param object rpcClient: SandboxStore service client (None by default)
:param object transferClient: client to upload/download sandboxes (None by default)
:param object smdb: SandboxMetadataDB object, or
True if SandboxMetadataDB is to be instantiated for direct access or
False if no direct access to the SandboxMetadataDB is done (default)
"""
self.__serviceName = "WorkloadManagement/SandboxStore"
self.__rpcClient = rpcClient
self.__transferClient = transferClient
self.__kwargs = kwargs
self.__vo = None
SandboxStoreClient.__smdb = smdb
if 'delegatedGroup' in kwargs:
self.__vo = getVOForGroup(kwargs['delegatedGroup'])
if SandboxStoreClient.__smdb is True:
try:
from DIRAC.WorkloadManagementSystem.DB.SandboxMetadataDB import SandboxMetadataDB
SandboxStoreClient.__smdb = SandboxMetadataDB()
result = SandboxStoreClient.__smdb._getConnection() # pylint: disable=protected-access
if not result['OK']:
SandboxStoreClient.__smdb = False
else:
result['Value'].close()
except (ImportError, RuntimeError, AttributeError):
SandboxStoreClient.__smdb = False
def __getUserDetails( self ):
""" Get details on user account
"""
credentials = self.getRemoteCredentials()
if credentials:
diracUser = credentials.get( "username" )
diracGroup = credentials.get( "group" )
if not ( diracUser and diracGroup ):
return S_ERROR( 'Failed to get DIRAC user name and/or group' )
vo = getVOForGroup( diracGroup )
diracHome = ''
if vo:
diracHome = '/%s/user/%s/%s' % ( vo, diracUser[0], diracUser )
cfgPath = self.serviceInfoDict[ 'serviceSectionPath' ]
gLogger.debug( "cfgPath: %s" % cfgPath )
irodsUser = gConfig.getValue( "%s/UserCredentials/%s/iRodsUser" % ( cfgPath , diracUser ) , diracUser )
irodsHome = gConfig.getValue( "%s/UserCredentials/%s/iRodsHome" % ( cfgPath , diracUser ) , '' )
irodsGroup = gConfig.getValue( "%s/UserCredentials/%s/iRodsGroup" % ( cfgPath , diracUser ) , '' )
irodsPassword = gConfig.getValue( "%s/UserCredentials/%s/iRodsPassword" % ( cfgPath , diracUser ) , '' )
resultDict = {}
resultDict['DIRACUser'] = diracUser
resultDict['DIRACGroup'] = diracGroup
resultDict['DIRACHome'] = diracHome
resultDict['iRodsUser'] = irodsUser
resultDict['iRodsGroup'] = irodsGroup
resultDict['iRodsHome'] = irodsHome
resultDict['iRodsPassword'] = irodsPassword
return S_OK( resultDict )
def __init__(self,
rpcClient=None,
transferClient=None,
smdb=None,
**kwargs):
self.__serviceName = "WorkloadManagement/SandboxStore"
self.__rpcClient = rpcClient
self.__transferClient = transferClient
self.__kwargs = kwargs
self.__vo = None
SandboxStoreClient.__smdb = smdb
if 'delegatedGroup' in kwargs:
self.__vo = getVOForGroup(kwargs['delegatedGroup'])
if SandboxStoreClient.__smdb is None:
try:
from DIRAC.WorkloadManagementSystem.DB.SandboxMetadataDB import SandboxMetadataDB
SandboxStoreClient.__smdb = SandboxMetadataDB()
result = SandboxStoreClient.__smdb._getConnection() # pylint: disable=protected-access
if not result['OK']:
SandboxStoreClient.__smdb = False
else:
result['Value'].close()
except (ImportError, RuntimeError, AttributeError):
SandboxStoreClient.__smdb = False
def __init__(self, *args, **kwargs):
super(JobLaunchpadHandler, self).__init__(*args, **kwargs)
sessionData = self.getSessionData()
for opt, value in (self.getAppSettings().get("Options") or {}).items():
self.defaultParams[opt] = value.replace(', ', ',').split(',')
self.user = sessionData['user'].get('username', '')
self.group = sessionData['user'].get('group', '')
self.vo = getVOForGroup(self.group)
def vo(self):
""":returns: return vo of the usergroup """
if self._vo:
return self._vo
if self.userGroup:
self._vo = getVOForGroup(self.userGroup)
return self._vo
def vo(self):
""":returns: return vo of the usergroup """
if self._vo:
return self._vo
if self.userGroup:
self._vo = getVOForGroup(self.userGroup)
return self._vo
def __init__(self, *args, **kwargs ):
super( TransferAccountingHandler, self ).__init__( *args, **kwargs )
sessionData = self.getSessionData()
self.user = sessionData['user'].get( 'username', '' )
self.group = sessionData['user'].get( 'group', '' )
self.vo = getVOForGroup( self.group )
self.fc = FileCatalog( vo = self.vo )
self.log.always(self.user)
self.log.always(self.group)
self.log.always(self.vo)
self.log.always(self.fc)
def getGroupScopes(self, group):
"""Get group scopes
:param str group: DIRAC group
:return: list
"""
idPScope = getGroupOption(group, "IdPRole")
if not idPScope:
idPScope = "wlcg.groups:/%s/%s" % (getVOForGroup(group),
group.split("_")[1])
return S_OK(scope_to_list(idPScope))
def getGroupScopes(self, group):
"""Get group scopes
:param str group: DIRAC group
:return: list
"""
idPScope = getGroupOption(group, "IdPRole")
if not idPScope:
idPScope = "eduperson_entitlement?value=urn:mace:egi.eu:group:%s:role=%s#aai.egi.eu" % (
getVOForGroup(group),
group.split("_")[1],
)
return S_OK(scope_to_list(idPScope))
def __init__( self, script = None, stdout = 'std.out', stderr = 'std.err' ):
"""Instantiates the Workflow object and some default parameters.
"""
self.log = gLogger
self.section = COMPONENT_NAME
self.dbg = False
if gConfig.getValue( self.section + '/LogLevel', 'DEBUG' ) == 'DEBUG':
self.dbg = True
#gConfig.getValue('Tier0SE-tape','SEName')
self.stepCount = 0
self.owner = 'NotSpecified'
self.name = 'Name'
self.type = 'User'
self.priority = 1
vo = ''
ret = getProxyInfo( disableVOMS = True )
if ret['OK'] and 'group' in ret['Value']:
vo = getVOForGroup( ret['Value']['group'] )
self.group = vo
self.site = 'ANY' #ANY
#self.setup = 'Development'
self.origin = 'DIRAC'
self.stdout = stdout
self.stderr = stderr
self.logLevel = 'info'
self.executable = '$DIRACROOT/scripts/dirac-jobexec' # to be clarified
self.addToInputSandbox = []
self.addToOutputSandbox = []
self.addToInputData = []
self.systemConfig = 'ANY'
self.reqParams = {'MaxCPUTime': 'other.NAME>=VALUE',
'MinCPUTime': 'other.NAME<=VALUE',
'Site': 'other.NAME=="VALUE"',
'Platform': 'other.NAME=="VALUE"',
#'BannedSites': '!Member(other.Site,BannedSites)', #doesn't work unfortunately
'BannedSites': 'other.Site!="VALUE"',
'SystemConfig': 'Member("VALUE",other.CompatiblePlatforms)'}
##Add member to handle Parametric jobs
self.parametric = {}
self.script = script
if not script:
self.workflow = Workflow()
self.__setJobDefaults()
else:
self.workflow = Workflow( script )
#Global error dictionary
self.errorDict = {}
def __init__( self, script = None, stdout = 'std.out', stderr = 'std.err' ):
"""Instantiates the Workflow object and some default parameters.
"""
self.log = gLogger
self.section = COMPONENT_NAME
self.dbg = False
if gConfig.getValue( self.section + '/LogLevel', 'DEBUG' ) == 'DEBUG':
self.dbg = True
#gConfig.getValue('Tier0SE-tape','SEName')
self.stepCount = 0
self.owner = 'NotSpecified'
self.name = 'Name'
self.type = 'User'
self.priority = 1
vo = ''
ret = getProxyInfo( disableVOMS = True )
if ret['OK'] and 'group' in ret['Value']:
vo = getVOForGroup( ret['Value']['group'] )
self.group = vo
self.site = 'ANY' #ANY
#self.setup = 'Development'
self.origin = 'DIRAC'
self.stdout = stdout
self.stderr = stderr
self.logLevel = 'info'
self.executable = '$DIRACROOT/scripts/dirac-jobexec' # to be clarified
self.addToInputSandbox = []
self.addToOutputSandbox = []
self.addToInputData = []
self.systemConfig = 'ANY'
self.reqParams = {'MaxCPUTime': 'other.NAME>=VALUE',
'MinCPUTime': 'other.NAME<=VALUE',
'Site': 'other.NAME=="VALUE"',
'Platform': 'other.NAME=="VALUE"',
#'BannedSites': '!Member(other.Site,BannedSites)', #doesn't work unfortunately
'BannedSites': 'other.Site!="VALUE"',
'SystemConfig': 'Member("VALUE",other.CompatiblePlatforms)'}
##Add member to handle Parametric jobs
self.parametric = {}
self.script = script
if not script:
self.workflow = Workflow()
self.__setJobDefaults()
else:
self.workflow = Workflow( script )
#Global error dictionary
self.errorDict = {}
def __init__(self, script=None, stdout='std.out', stderr='std.err'):
"""Instantiates the Workflow object and some default parameters.
"""
super(Job, self).__init__()
self.stepCount = 0
self.owner = 'NotSpecified'
self.name = 'Name'
self.type = 'User'
self.priority = 1
vo = ''
ret = getProxyInfo(disableVOMS=True)
if ret['OK'] and 'group' in ret['Value']:
vo = getVOForGroup(ret['Value']['group'])
self.group = vo
self.site = None
# self.setup = 'Development'
self.origin = 'DIRAC'
self.stdout = stdout
self.stderr = stderr
self.logLevel = 'info'
self.executable = '$DIRACROOT/scripts/dirac-jobexec' # to be clarified
# $DIRACROOT is set by the JobWrapper at execution time
self.addToInputSandbox = []
self.addToOutputSandbox = []
self.addToInputData = []
# #Add member to handle Parametric jobs
self.numberOfParameters = 0
self.parameterSeqs = {}
self.wfArguments = {}
self.parametricWFArguments = {}
# loading the function that will be used to determine the platform (it can be VO specific)
res = ObjectLoader().loadObject("ConfigurationSystem.Client.Helpers.Resources", 'getDIRACPlatforms')
if not res['OK']:
self.log.fatal(res['Message'])
self.getDIRACPlatforms = res['Value']
self.script = script
if not script:
self.workflow = Workflow()
self.__setJobDefaults()
else:
self.workflow = Workflow(script)
self._siteSet = set(getSites().get('Value', []))
def __init__(self, script=None, stdout='std.out', stderr='std.err'):
"""Instantiates the Workflow object and some default parameters.
"""
super(Job, self).__init__()
self.stepCount = 0
self.owner = 'NotSpecified'
self.name = 'Name'
self.type = 'User'
self.priority = 1
vo = ''
ret = getProxyInfo(disableVOMS=True)
if ret['OK'] and 'group' in ret['Value']:
vo = getVOForGroup(ret['Value']['group'])
self.group = vo
self.site = None
# self.setup = 'Development'
self.origin = 'DIRAC'
self.stdout = stdout
self.stderr = stderr
self.logLevel = 'info'
self.executable = '$DIRACROOT/scripts/dirac-jobexec' # to be clarified
# $DIRACROOT is set by the JobWrapper at execution time
self.addToInputSandbox = []
self.addToOutputSandbox = []
self.addToInputData = []
# #Add member to handle Parametric jobs
self.numberOfParameters = 0
self.parameterSeqs = {}
self.wfArguments = {}
self.parametricWFArguments = {}
# loading the function that will be used to determine the platform (it can be VO specific)
res = ObjectLoader().loadObject("ConfigurationSystem.Client.Helpers.Resources", 'getDIRACPlatforms')
if not res['OK']:
self.log.fatal(res['Message'])
self.getDIRACPlatforms = res['Value']
self.script = script
if not script:
self.workflow = Workflow()
self.__setJobDefaults()
else:
self.workflow = Workflow(script)
self._siteSet = set(getSites().get('Value', []))
def __init__( self, script = None, stdout = 'std.out', stderr = 'std.err' ):
"""Instantiates the Workflow object and some default parameters.
"""
super( Job, self ).__init__()
self.dbg = False
if gConfig.getValue( self.section + '/LogLevel', 'DEBUG' ) == 'DEBUG':
self.dbg = True
#gConfig.getValue('Tier0SE-tape','SEName')
self.stepCount = 0
self.owner = 'NotSpecified'
self.name = 'Name'
self.type = 'User'
self.priority = 1
vo = ''
ret = getProxyInfo( disableVOMS = True )
if ret['OK'] and 'group' in ret['Value']:
vo = getVOForGroup( ret['Value']['group'] )
self.group = vo
self.site = 'ANY' #ANY
#self.setup = 'Development'
self.origin = 'DIRAC'
self.stdout = stdout
self.stderr = stderr
self.logLevel = 'info'
self.executable = '$DIRACROOT/scripts/dirac-jobexec' # to be clarified
self.addToInputSandbox = []
self.addToOutputSandbox = []
self.addToInputData = []
##Add member to handle Parametric jobs
self.numberOfParameters = 0
self.parameterSeqs = {}
self.wfArguments = {}
self.parametricWFArguments = {}
self.script = script
if not script:
self.workflow = Workflow()
self.__setJobDefaults()
else:
self.workflow = Workflow( script )
def __init__(self, rpcClient=None, transferClient=None, **kwargs):
self.__serviceName = "WorkloadManagement/SandboxStore"
self.__rpcClient = rpcClient
self.__transferClient = transferClient
self.__kwargs = kwargs
self.__vo = None
if 'delegatedGroup' in kwargs:
self.__vo = getVOForGroup(kwargs['delegatedGroup'])
if SandboxStoreClient.__smdb is None:
try:
from DIRAC.WorkloadManagementSystem.DB.SandboxMetadataDB import SandboxMetadataDB
SandboxStoreClient.__smdb = SandboxMetadataDB()
result = SandboxStoreClient.__smdb._getConnection() # pylint: disable=protected-access
if not result['OK']:
SandboxStoreClient.__smdb = False
else:
result['Value'].close()
except (ImportError, RuntimeError, AttributeError):
SandboxStoreClient.__smdb = False
def parseSwitches():
'''
Parses the arguments passed by the user
'''
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if not len(args) == 2:
Script.showHelp()
params = {}
params['ce'] = None
params['site'] = None
params['host'] = None
params['vo'] = None
params['info'] = args[0]
ret = getProxyInfo(disableVOMS=True)
if ret['OK'] and 'group' in ret['Value']:
params['vo'] = getVOForGroup(ret['Value']['group'])
else:
Script.gLogger.error('Could not determine VO')
Script.showHelp()
if params['info'] in ['ce', 'ce-state', 'ce-cluster', 'ce-vo']:
params['ce'] = args[1]
elif params['info']in ['site', 'site-se']:
params['site'] = args[1]
else:
Script.gLogger.error('Wrong argument value')
Script.showHelp()
for unprocSw in Script.getUnprocessedSwitches():
if unprocSw[0] in ("H", "host"):
params['host'] = unprocSw[1]
if unprocSw[0] in ("V", "vo"):
params['vo'] = unprocSw[1]
return params
def parseSwitches():
'''
Parses the arguments passed by the user
'''
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if not len(args) == 2:
Script.showHelp()
params = {}
params['ce'] = None
params['site'] = None
params['host'] = None
params['vo'] = None
params['info'] = args[0]
ret = getProxyInfo(disableVOMS=True)
if ret['OK'] and 'group' in ret['Value']:
params['vo'] = getVOForGroup(ret['Value']['group'])
else:
Script.gLogger.error('Could not determine VO')
Script.showHelp()
if params['info'] in ['ce', 'ce-state', 'ce-cluster', 'ce-vo']:
params['ce'] = args[1]
elif params['info'] in ['site', 'site-se']:
params['site'] = args[1]
else:
Script.gLogger.error('Wrong argument value')
Script.showHelp()
for unprocSw in Script.getUnprocessedSwitches():
if unprocSw[0] in ("H", "host"):
params['host'] = unprocSw[1]
if unprocSw[0] in ("V", "vo"):
params['vo'] = unprocSw[1]
return params
def __init__(self, script=None, stdout="std.out", stderr="std.err"):
"""Instantiates the Workflow object and some default parameters.
"""
super(Job, self).__init__()
self.dbg = False
if gConfig.getValue(self.section + "/LogLevel", "DEBUG") == "DEBUG":
self.dbg = True
# gConfig.getValue('Tier0SE-tape','SEName')
self.stepCount = 0
self.owner = "NotSpecified"
self.name = "Name"
self.type = "User"
self.priority = 1
vo = ""
ret = getProxyInfo(disableVOMS=True)
if ret["OK"] and "group" in ret["Value"]:
vo = getVOForGroup(ret["Value"]["group"])
self.group = vo
self.site = "ANY" # ANY
# self.setup = 'Development'
self.origin = "DIRAC"
self.stdout = stdout
self.stderr = stderr
self.logLevel = "info"
self.executable = "$DIRACROOT/scripts/dirac-jobexec" # to be clarified
self.addToInputSandbox = []
self.addToOutputSandbox = []
self.addToInputData = []
##Add member to handle Parametric jobs
self.parametric = {}
self.script = script
if not script:
self.workflow = Workflow()
self.__setJobDefaults()
else:
self.workflow = Workflow(script)
def __init__( self, script = None, stdout = 'std.out', stderr = 'std.err' ):
"""Instantiates the Workflow object and some default parameters.
"""
super( Job, self ).__init__()
self.dbg = False
if gConfig.getValue( self.section + '/LogLevel', 'DEBUG' ) == 'DEBUG':
self.dbg = True
#gConfig.getValue('Tier0SE-tape','SEName')
self.stepCount = 0
self.owner = 'NotSpecified'
self.name = 'Name'
self.type = 'User'
self.priority = 1
vo = ''
ret = getProxyInfo( disableVOMS = True )
if ret['OK'] and 'group' in ret['Value']:
vo = getVOForGroup( ret['Value']['group'] )
self.group = vo
self.site = 'ANY' #ANY
#self.setup = 'Development'
self.origin = 'DIRAC'
self.stdout = stdout
self.stderr = stderr
self.logLevel = 'info'
self.executable = '$DIRACROOT/scripts/dirac-jobexec' # to be clarified
self.addToInputSandbox = []
self.addToOutputSandbox = []
self.addToInputData = []
##Add member to handle Parametric jobs
self.parametric = {}
self.script = script
if not script:
self.workflow = Workflow()
self.__setJobDefaults()
else:
self.workflow = Workflow( script )
def input_path_to_lfn(input_path):
res = getProxyInfo(False, False)
if not res['OK']:
gLogger.error("Failed to get client proxy information.",
res['Message'])
DIRAC.exit(2)
proxyInfo = res['Value']
if proxyInfo['secondsLeft'] == 0:
gLogger.error("Proxy expired")
DIRAC.exit(2)
username = proxyInfo['username']
vo = ''
if 'group' in proxyInfo:
vo = getVOForGroup(proxyInfo['group'])
folder_name = os.path.basename(os.path.dirname(input_path))
lfn = '/%s/user/%s/%s/jsub/' % (
vo, username[0],
username) + folder_name + '/' + os.path.basename(input_path)
gLogger.debug('input path = %s' % input_path)
gLogger.debug('lfn translation = %s' % lfn)
return (lfn)
def main():
days = 0
months = 0
years = 0
wildcard = None
baseDir = ''
emptyDirsFlag = False
Script.registerSwitch("D:", "Days=",
"Match files older than number of days [%s]" % days)
Script.registerSwitch(
"M:", "Months=",
"Match files older than number of months [%s]" % months)
Script.registerSwitch(
"Y:", "Years=", "Match files older than number of years [%s]" % years)
Script.registerSwitch("w:", "Wildcard=",
"Wildcard for matching filenames [All]")
Script.registerSwitch(
"b:", "BaseDir=",
"Base directory to begin search (default /[vo]/user/[initial]/[username])"
)
Script.registerSwitch("e", "EmptyDirs",
"Create a list of empty directories")
Script.parseCommandLine(ignoreErrors=False)
for switch in Script.getUnprocessedSwitches():
if switch[0] == "D" or switch[0].lower() == "days":
days = int(switch[1])
if switch[0] == "M" or switch[0].lower() == "months":
months = int(switch[1])
if switch[0] == "Y" or switch[0].lower() == "years":
years = int(switch[1])
if switch[0].lower() == "w" or switch[0].lower() == "wildcard":
wildcard = '*' + switch[1]
if switch[0].lower() == "b" or switch[0].lower() == "basedir":
baseDir = switch[1]
if switch[0].lower() == "e" or switch[0].lower() == "emptydirs":
emptyDirsFlag = True
import DIRAC
from DIRAC import gLogger
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from datetime import datetime, timedelta
import sys
import os
import time
import fnmatch
fc = FileCatalog()
def isOlderThan(cTimeStruct, days):
timeDelta = timedelta(days=days)
maxCTime = datetime.utcnow() - timeDelta
if cTimeStruct < maxCTime:
return True
return False
withMetadata = False
if days or months or years:
withMetadata = True
totalDays = 0
if years:
totalDays += 365 * years
if months:
totalDays += 30 * months
if days:
totalDays += days
res = getProxyInfo(False, False)
if not res['OK']:
gLogger.error("Failed to get client proxy information.",
res['Message'])
DIRAC.exit(2)
proxyInfo = res['Value']
if proxyInfo['secondsLeft'] == 0:
gLogger.error("Proxy expired")
DIRAC.exit(2)
username = proxyInfo['username']
vo = ''
if 'group' in proxyInfo:
vo = getVOForGroup(proxyInfo['group'])
if not baseDir:
if not vo:
gLogger.error('Could not determine VO')
Script.showHelp()
baseDir = '/%s/user/%s/%s' % (vo, username[0], username)
baseDir = baseDir.rstrip('/')
gLogger.notice('Will search for files in %s%s' %
(baseDir, (' matching %s' % wildcard) if wildcard else ''))
activeDirs = [baseDir]
allFiles = []
emptyDirs = []
while len(activeDirs) > 0:
currentDir = activeDirs.pop()
res = fc.listDirectory(currentDir, withMetadata, timeout=360)
if not res['OK']:
gLogger.error("Error retrieving directory contents",
"%s %s" % (currentDir, res['Message']))
elif currentDir in res['Value']['Failed']:
gLogger.error(
"Error retrieving directory contents",
"%s %s" % (currentDir, res['Value']['Failed'][currentDir]))
else:
dirContents = res['Value']['Successful'][currentDir]
subdirs = dirContents['SubDirs']
files = dirContents['Files']
if not subdirs and not files:
emptyDirs.append(currentDir)
gLogger.notice('%s: empty directory' % currentDir)
else:
for subdir in sorted(subdirs, reverse=True):
if (not withMetadata) or isOlderThan(
subdirs[subdir]['CreationDate'], totalDays):
activeDirs.append(subdir)
for filename in sorted(files):
fileOK = False
if (not withMetadata) or isOlderThan(
files[filename]['MetaData']['CreationDate'],
totalDays):
if wildcard is None or fnmatch.fnmatch(
filename, wildcard):
fileOK = True
if not fileOK:
files.pop(filename)
allFiles += sorted(files)
if len(files) or len(subdirs):
gLogger.notice(
"%s: %d files%s, %d sub-directories" %
(currentDir, len(files), ' matching'
if withMetadata or wildcard else '', len(subdirs)))
outputFileName = '%s.lfns' % baseDir.replace('/%s' % vo,
'%s' % vo).replace('/', '-')
outputFile = open(outputFileName, 'w')
for lfn in sorted(allFiles):
outputFile.write(lfn + '\n')
outputFile.close()
gLogger.notice('%d matched files have been put in %s' %
(len(allFiles), outputFileName))
if emptyDirsFlag:
outputFileName = '%s.emptydirs' % baseDir.replace(
'/%s' % vo, '%s' % vo).replace('/', '-')
outputFile = open(outputFileName, 'w')
for dir in sorted(emptyDirs):
outputFile.write(dir + '\n')
outputFile.close()
gLogger.notice('%d empty directories have been put in %s' %
(len(emptyDirs), outputFileName))
DIRAC.exit(0)
totalDays += 30 * months
if days:
totalDays += days
res = getProxyInfo( False, False )
if not res['OK']:
gLogger.error( "Failed to get client proxy information.", res['Message'] )
DIRAC.exit( 2 )
proxyInfo = res['Value']
if proxyInfo['secondsLeft'] == 0:
gLogger.error( "Proxy expired" )
DIRAC.exit( 2 )
username = proxyInfo['username']
vo = ''
if 'group' in proxyInfo:
vo = getVOForGroup( proxyInfo['group'] )
if not baseDir:
if not vo:
gLogger.error( 'Could not determine VO' )
Script.showHelp()
baseDir = '/%s/user/%s/%s' % ( vo, username[0], username )
baseDir = baseDir.rstrip( '/' )
gLogger.info( 'Will search for files in %s' % baseDir )
activeDirs = [baseDir]
allFiles = []
emptyDirs = []
while len( activeDirs ) > 0:
currentDir = activeDirs.pop()
# These variables are defined by the setup fixture
local_path = putDir = createDir = putFile = isFile = listDir = getFile = rmDir = None
getDir = removeFile = filesInFolderAandB = fileAdlers = fileSizes = None
se = None
try:
res = getProxyInfo()
if not res["OK"]:
gLogger.error("Failed to get client proxy information.",
res["Message"])
sys.exit(2)
proxyInfo = res["Value"]
username = proxyInfo["username"]
vo = ""
if "group" in proxyInfo:
vo = getVOForGroup(proxyInfo["group"])
DESTINATION_PATH = "/%s/user/%s/%s/gfaltests" % (vo, username[0], username)
except Exception as e: # pylint: disable=broad-except
print(repr(e))
sys.exit(2)
# local path containing test files. There should be a folder called Workflow containing
# (the files can be simple textfiles)
# FolderA
# -FolderAA
# --FileAA
# -FileA
# FolderB
# -FileB
records.append(('Setup', gConfig.getValue('/DIRAC/Setup', 'Unknown')))
records.append(('ConfigurationServer', gConfig.getValue('/DIRAC/Configuration/Servers', [])))
records.append(('Installation path', DIRAC.rootPath))
if os.path.exists(os.path.join(DIRAC.rootPath, DIRAC.getPlatform(), 'bin', 'mysql')):
records.append(('Installation type', 'server'))
else:
records.append(('Installation type', 'client'))
records.append(('Platform', DIRAC.getPlatform()))
ret = getProxyInfo(disableVOMS=True)
if ret['OK']:
if 'group' in ret['Value']:
vo = getVOForGroup(ret['Value']['group'])
else:
vo = getVOForGroup('')
if not vo:
vo = "None"
records.append(('VirtualOrganization', vo))
if 'identity' in ret['Value']:
records.append(('User DN', ret['Value']['identity']))
if 'secondsLeft' in ret['Value']:
records.append(('Proxy validity, secs', {'Value': str(ret['Value']['secondsLeft']), 'Just': 'L'}))
if gConfig.getValue('/DIRAC/Security/UseServerCertificate', True):
records.append(('Use Server Certificate', 'Yes'))
else:
records.append(('Use Server Certificate', 'No'))
if gConfig.getValue('/DIRAC/Security/SkipCAChecks', False):
str(gConfig.getValue('/DIRAC/Configuration/Servers', []))))
records.append(('Installation path', DIRAC.rootPath))
if os.path.exists(
os.path.join(DIRAC.rootPath, DIRAC.getPlatform(), 'bin', 'mysql')):
records.append(('Installation type', 'server'))
else:
records.append(('Installation type', 'client'))
records.append(('Platform', DIRAC.getPlatform()))
ret = getProxyInfo(disableVOMS=True)
if ret['OK']:
if 'group' in ret['Value']:
records.append(
('VirtualOrganization', getVOForGroup(ret['Value']['group'])))
else:
records.append(('VirtualOrganization', getVOForGroup('')))
if 'identity' in ret['Value']:
records.append(('User DN', ret['Value']['identity']))
if 'secondsLeft' in ret['Value']:
records.append(
('Proxy validity, secs', str(ret['Value']['secondsLeft'])))
if gConfig.getValue('/DIRAC/Security/UseServerCertificate', True):
records.append(('Use Server Certificate', 'Yes'))
else:
records.append(('Use Server Certificate', 'No'))
if gConfig.getValue('/DIRAC/Security/SkipCAChecks', False):
records.append(('Skip CA Checks', 'Yes'))
else:
def _checkFilesToStage( seToLFNs, onlineLFNs, offlineLFNs, absentLFNs,
checkOnlyTapeSEs = None, jobLog = None,
proxyUserName = None,
proxyUserGroup = None,
executionLock = None ):
"""
Checks on SEs whether the file is NEARLINE or ONLINE
onlineLFNs, offlineLFNs and absentLFNs are modified to contain the files found online
If checkOnlyTapeSEs is True, disk replicas are not checked
As soon as a replica is found Online for a file, no further check is made
"""
# Only check on storage if it is a tape SE
if jobLog is None:
logger = gLogger
else:
logger = jobLog
if checkOnlyTapeSEs is None:
# Default value is True
checkOnlyTapeSEs = True
failed = {}
for se, lfnsInSEList in seToLFNs.iteritems():
# If we have found already all files online at another SE, no need to check the others
# but still we want to set the SE as Online if not a TapeSE
vo = getVOForGroup( proxyUserGroup )
seObj = StorageElement( se, vo = vo )
status = seObj.getStatus()
if not status['OK']:
return status
tapeSE = status['Value']['TapeSE']
diskSE = status['Value']['DiskSE']
# If requested to check only Tape SEs and the file is at a diskSE, we guess it is Online...
filesToCheck = []
for lfn in lfnsInSEList:
# If the file had already been found accessible at an SE, only check that this one is on disk
diskIsOK = checkOnlyTapeSEs or ( lfn in onlineLFNs )
if diskIsOK and diskSE:
onlineLFNs.setdefault( lfn, [] ).append( se )
elif not diskIsOK or ( tapeSE and ( lfn not in onlineLFNs ) ):
filesToCheck.append( lfn )
if not filesToCheck:
continue
# We have to use a new SE object because it caches the proxy!
with UserProxy(proxyUserName=proxyUserName,
proxyUserGroup=proxyUserGroup,
executionLock=executionLock) as proxyResult:
if proxyResult['OK']:
fileMetadata = StorageElement(se, vo=vo).getFileMetadata(filesToCheck)
else:
fileMetadata = proxyResult
if not fileMetadata['OK']:
failed[se] = dict.fromkeys( filesToCheck, fileMetadata['Message'] )
else:
if fileMetadata['Value']['Failed']:
failed[se] = fileMetadata['Value']['Failed']
# is there at least one replica online?
for lfn, mDict in fileMetadata['Value']['Successful'].iteritems():
# SRM returns Cached, but others may only return Accessible
if mDict.get( 'Cached', mDict['Accessible'] ):
onlineLFNs.setdefault( lfn, [] ).append( se )
elif tapeSE:
# A file can be staged only at Tape SE
offlineLFNs.setdefault( lfn, [] ).append( se )
else:
# File not available at a diskSE... we shall retry later
pass
# Doesn't matter if some files are Offline if they are also online
for lfn in set( offlineLFNs ) & set( onlineLFNs ):
offlineLFNs.pop( lfn )
# If the file was found staged, ignore possible errors, but print out errors
for se, failedLfns in failed.items():
logger.error( "Errors when getting files metadata", 'at %s' % se )
for lfn, reason in failedLfns.items():
if lfn in onlineLFNs:
logger.warn( reason, 'for %s, but there is an online replica' % lfn )
failed[se].pop( lfn )
else:
logger.error( reason, 'for %s, no online replicas' % lfn )
if cmpError( reason, errno.ENOENT ):
absentLFNs.setdefault( lfn, [] ).append( se )
failed[se].pop( lfn )
if not failed[se]:
failed.pop( se )
# Find the files that do not exist at SE
if failed:
logger.error( "Error getting metadata", "for %d files" % len( set( lfn for lfnList in failed.itervalues() for lfn in lfnList ) ) )
for lfn in absentLFNs:
seList = absentLFNs[lfn]
# FIXME: it is not possible to return here an S_ERROR(), return the message only
absentLFNs[lfn] = S_ERROR( errno.ENOENT, "File not at %s" % ','.join( seList ) )['Message']
# Format the error for absent files
return S_OK()
def __init__(self, *args, **kwargs ): super( JobLaunchpadHandler, self ).__init__( *args, **kwargs ) sessionData = self.getSessionData() self.user = sessionData['user'].get( 'username', '' ) self.group = sessionData['user'].get( 'group', '' ) self.vo = getVOForGroup( self.group )
def main():
dm = DataManager()
fileTupleBuffer = []
res = getProxyInfo( False, False )
if not res['OK']:
gLogger.error( "Failed to get client proxy information.", res['Message'] )
DIRAC.exit( 2 )
proxyInfo = res['Value']
if proxyInfo['secondsLeft'] == 0:
gLogger.error( "Proxy expired" )
DIRAC.exit( 2 )
username = proxyInfo['username']
vo = ''
if 'group' in proxyInfo:
vo = getVOForGroup( proxyInfo['group'] )
counter = 0
for f in files:
counter += 1
local_f=f
if not f.startswith('/cefs'):
# gLogger.error('File must be under "/cefs"')
# continue
#if the file to reg is not under /cefs, use put and register
folder_name=os.path.basename(os.path.dirname(f))
# lfn = '/cepc/user/%s/%s/jsub/'%(username[0],username) + folder_name + '/' + os.path.basename(f)
lfn = '/cepc/user/%s/%s/jsub/'%(username[0],username) + folder_name + '/' + os.path.basename(f)
# dirname = os.path.dirname(local_f)
# os.system('mkdir -p %s'%(dirname))
# os.system('cp %s %s' %(f,local_f))
do_put_and_register=True
else:
lfn = '/cepc/lustre-ro' + os.path.abspath(f)
do_put_and_register=False
result = fcc.isFile(lfn)
if result['OK'] and lfn in result['Value']['Successful'] and result['Value']['Successful'][lfn]:
continue
size = os.path.getsize(f)
adler32 = fileAdler(f)
guid = makeGuid()
fileTuple = (lfn, local_f, size, _se, guid, adler32)
fileTupleBuffer.append(fileTuple)
gLogger.debug('Register to lfn: %s' % lfn)
gLogger.debug('fileTuple: %s' % (fileTuple,))
if len(fileTupleBuffer) >= _bufferSize:
if do_put_and_register:
result = dm.putAndRegister(lfn, local_f, _se, guid, overwrite=overwrite)
else:
result = dm.registerFile(fileTupleBuffer)
print('register result', result)
# if not result['OK']:
# gLogger.error('Register file failed')
# return 1
del fileTupleBuffer[:]
gLogger.debug('%s files registered' % counter)
if fileTupleBuffer:
if do_put_and_register:
result = dm.putAndRegister(lfn, local_f, _se, guid, overwrite=overwrite)
else:
result = dm.registerFile(fileTupleBuffer)
print('register result', result)
# if not result['OK']:
# gLogger.error('Register file failed')
# return 1
del fileTupleBuffer[:]
gLogger.info('Totally %s files registered' % counter)
return 0
records.append(('Setup', gConfig.getValue('/DIRAC/Setup', 'Unknown')))
records.append(('ConfigurationServer', gConfig.getValue('/DIRAC/Configuration/Servers', [])))
records.append(('Installation path', DIRAC.rootPath))
if os.path.exists(os.path.join(DIRAC.rootPath, DIRAC.getPlatform(), 'bin', 'mysql')):
records.append(('Installation type', 'server'))
else:
records.append(('Installation type', 'client'))
records.append(('Platform', DIRAC.getPlatform()))
ret = getProxyInfo(disableVOMS=True)
if ret['OK']:
if 'group' in ret['Value']:
vo = getVOForGroup(ret['Value']['group'])
else:
vo = getVOForGroup('')
if not vo:
vo = "None"
records.append(('VirtualOrganization', vo))
if 'identity' in ret['Value']:
records.append(('User DN', ret['Value']['identity']))
if 'secondsLeft' in ret['Value']:
records.append(('Proxy validity, secs', {'Value': str(ret['Value']['secondsLeft']), 'Just': 'L'}))
if gConfig.getValue('/DIRAC/Security/UseServerCertificate', True):
records.append(('Use Server Certificate', 'Yes'))
else:
records.append(('Use Server Certificate', 'No'))
if gConfig.getValue('/DIRAC/Security/SkipCAChecks', False):
def setOutputData(self, lfns, OutputPath='', OutputSE=['']):
"""Helper function, used in preference to Job.setOutputData() for ILC.
For specifying output data to be registered in Grid storage. If a list
of OutputSEs are specified the job wrapper will try each in turn until
successful.
Example usage:
>>> job = Job()
>>> job.setOutputData(['Ntuple.root'])
@param lfns: Output data file or files
@type lfns: Single string or list of strings ['','']
@param OutputSE: Optional parameter to specify the Storage
@param OutputPath: Optional parameter to specify the Path in the Storage, postpented to /vo/user/u/username/
Element to store data or files, e.g. CERN-tape
@type OutputSE: string or list
@type OutputPath: string
"""
kwargs = {'lfns': lfns, 'OutputSE': OutputSE, 'OutputPath': OutputPath}
if type(lfns) == list and len(lfns):
outputDataStr = ';'.join(lfns)
description = 'List of output data files'
self._addParameter(self.workflow, 'UserOutputData', 'JDL',
outputDataStr, description)
elif type(lfns) == type(" "):
description = 'Output data file'
self._addParameter(self.workflow, 'UserOutputData', 'JDL', lfns,
description)
else:
return self._reportError(
'Expected file name string or list of file names for output data',
**kwargs)
if OutputSE:
description = 'User specified Output SE'
if type(OutputSE) in types.StringTypes:
OutputSE = [OutputSE]
elif type(OutputSE) != types.ListType:
return self._reportError(
'Expected string or list for OutputSE', **kwargs)
OutputSE = ';'.join(OutputSE)
self._addParameter(self.workflow, 'UserOutputSE', 'JDL', OutputSE,
description)
if OutputPath:
description = 'User specified Output Path'
if not type(OutputPath) in types.StringTypes:
return self._reportError('Expected string for OutputPath',
**kwargs)
# Remove leading "/" that might cause problems with os.path.join
while OutputPath[0] == '/':
OutputPath = OutputPath[1:]
while OutputPath.count("//"):
OutputPath.replace('//', '/')
vo = getVOForGroup(self.proxyinfo['Value']['group'])
vostring = '/%s/user' % vo
if OutputPath.count(vostring):
return self._reportError(
'Output path contains %s which is not what you want' %
vostring, **kwargs)
self._addParameter(self.workflow, 'UserOutputPath', 'JDL',
OutputPath, description)
return S_OK()
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager
from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient
from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient
from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus
from DIRAC.ResourceStatusSystem.Client.SiteStatus import SiteStatus
from DIRAC.WorkloadManagementSystem.Client.JobManagerClient import JobManagerClient
from DIRAC.WorkloadManagementSystem.Client.WMSAdministratorClient import WMSAdministratorClient
from DIRAC.WorkloadManagementSystem.Client.PilotManagerClient import PilotManagerClient
voName = ""
ret = getProxyInfo(disableVOMS=True)
if ret["OK"] and "group" in ret["Value"]:
voName = getVOForGroup(ret["Value"]["group"])
COMPONENT_NAME = "/Interfaces/API/DiracAdmin"
class DiracAdmin(API):
"""Administrative functionalities"""
#############################################################################
def __init__(self):
"""Internal initialization of the DIRAC Admin API."""
super(DiracAdmin, self).__init__()
self.csAPI = CSAPI()
self.dbg = False
def selectJob( self, resourceDescription ):
""" Main job selection function to find the highest priority job
matching the resource capacity
"""
startTime = time.time()
# Check and form the resource description dictionary
resourceDict = {}
if type( resourceDescription ) in StringTypes:
classAdAgent = ClassAd( resourceDescription )
if not classAdAgent.isOK():
return S_ERROR( 'Illegal Resource JDL' )
gLogger.verbose( classAdAgent.asJDL() )
for name in gTaskQueueDB.getSingleValueTQDefFields():
if classAdAgent.lookupAttribute( name ):
if name == 'CPUTime':
resourceDict[name] = classAdAgent.getAttributeInt( name )
else:
resourceDict[name] = classAdAgent.getAttributeString( name )
for name in gTaskQueueDB.getMultiValueMatchFields():
if classAdAgent.lookupAttribute( name ):
resourceDict[name] = classAdAgent.getAttributeString( name )
# Check if a JobID is requested
if classAdAgent.lookupAttribute( 'JobID' ):
resourceDict['JobID'] = classAdAgent.getAttributeInt( 'JobID' )
if classAdAgent.lookupAttribute( 'DIRACVersion' ):
resourceDict['DIRACVersion'] = classAdAgent.getAttributeString( 'DIRACVersion' )
if classAdAgent.lookupAttribute( 'VirtualOrganization' ):
resourceDict['VirtualOrganization'] = classAdAgent.getAttributeString( 'VirtualOrganization' )
else:
for name in gTaskQueueDB.getSingleValueTQDefFields():
if resourceDescription.has_key( name ):
resourceDict[name] = resourceDescription[name]
for name in gTaskQueueDB.getMultiValueMatchFields():
if resourceDescription.has_key( name ):
resourceDict[name] = resourceDescription[name]
if resourceDescription.has_key( 'JobID' ):
resourceDict['JobID'] = resourceDescription['JobID']
if resourceDescription.has_key( 'DIRACVersion' ):
resourceDict['DIRACVersion'] = resourceDescription['DIRACVersion']
if resourceDescription.has_key( 'VirtualOrganization' ):
resourceDict['VirtualOrganization'] = resourceDescription['VirtualOrganization']
# Check the pilot DIRAC version
if self.checkPilotVersion:
if not 'DIRACVersion' in resourceDict:
return S_ERROR( 'Version check requested and not provided by Pilot' )
# Check if the matching Request provides a VirtualOrganization
if 'VirtualOrganization' in resourceDict:
voName = resourceDict['VirtualOrganization']
# Check if the matching Request provides an OwnerGroup
elif 'OwnerGroup' in resourceDict:
voName = getVOForGroup( resourceDict['OwnerGroup'] )
# else take the default VirtualOrganization for the installation
else:
voName = getVOForGroup( '' )
self.pilotVersion = gConfig.getValue( '/Operations/%s/%s/Versions/PilotVersion' % ( voName, self.setup ), '' )
if self.pilotVersion and resourceDict['DIRACVersion'] != self.pilotVersion:
return S_ERROR( 'Pilot version does not match the production version %s:%s' % \
( resourceDict['DIRACVersion'], self.pilotVersion ) )
# Get common site mask and check the agent site
result = gJobDB.getSiteMask( siteState = 'Active' )
if result['OK']:
maskList = result['Value']
else:
return S_ERROR( 'Internal error: can not get site mask' )
if not 'Site' in resourceDict:
return S_ERROR( 'Missing Site Name in Resource JDL' )
siteName = resourceDict['Site']
if resourceDict['Site'] not in maskList:
if 'GridCE' in resourceDict:
del resourceDict['Site']
else:
return S_ERROR( 'Site not in mask and GridCE not specified' )
resourceDict['Setup'] = self.serviceInfoDict['clientSetup']
if DEBUG:
print "Resource description:"
for key, value in resourceDict.items():
print key.rjust( 20 ), value
# Check if Job Limits are imposed onto the site
extraConditions = {}
if self.siteJobLimits:
result = self.getExtraConditions( siteName )
if result['OK']:
extraConditions = result['Value']
if extraConditions:
gLogger.info( 'Job Limits for site %s are: %s' % ( siteName, str( extraConditions ) ) )
result = gTaskQueueDB.matchAndGetJob( resourceDict, extraConditions = extraConditions )
if DEBUG:
print result
if not result['OK']:
return result
result = result['Value']
if not result['matchFound']:
return S_ERROR( 'No match found' )
jobID = result['jobId']
resAtt = gJobDB.getJobAttributes( jobID, ['OwnerDN', 'OwnerGroup', 'Status'] )
if not resAtt['OK']:
return S_ERROR( 'Could not retrieve job attributes' )
if not resAtt['Value']:
return S_ERROR( 'No attributes returned for job' )
if not resAtt['Value']['Status'] == 'Waiting':
gLogger.error( 'Job %s matched by the TQ is not in Waiting state' % str( jobID ) )
result = gTaskQueueDB.deleteJob( jobID )
result = gJobDB.setJobStatus( jobID, status = 'Matched', minor = 'Assigned' )
result = gJobLoggingDB.addLoggingRecord( jobID,
status = 'Matched',
minor = 'Assigned',
source = 'Matcher' )
result = gJobDB.getJobJDL( jobID )
if not result['OK']:
return S_ERROR( 'Failed to get the job JDL' )
resultDict = {}
resultDict['JDL'] = result['Value']
resultDict['JobID'] = jobID
matchTime = time.time() - startTime
gLogger.info( "Match time: [%s]" % str( matchTime ) )
gMonitor.addMark( "matchTime", matchTime )
# Get some extra stuff into the response returned
resOpt = gJobDB.getJobOptParameters( jobID )
if resOpt['OK']:
for key, value in resOpt['Value'].items():
resultDict[key] = value
resAtt = gJobDB.getJobAttributes( jobID, ['OwnerDN', 'OwnerGroup'] )
if not resAtt['OK']:
return S_ERROR( 'Could not retrieve job attributes' )
if not resAtt['Value']:
return S_ERROR( 'No attributes returned for job' )
resultDict['DN'] = resAtt['Value']['OwnerDN']
resultDict['Group'] = resAtt['Value']['OwnerGroup']
return S_OK( resultDict )
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup
Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
'Usage:',
' %s [option|cfgfile] ... Site' % Script.scriptName, ] ) )
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
infoDict = {}
infoDict['Setup'] = gConfig.getValue( '/DIRAC/Setup', 'Unknown' )
infoDict['ConfigurationServer'] = gConfig.getValue( '/DIRAC/Configuration/Servers', [] )
ret = getProxyInfo( disableVOMS = True )
if ret['OK'] and 'group' in ret['Value']:
infoDict['VirtualOrganization'] = getVOForGroup( ret['Value']['group'] )
else:
infoDict['VirtualOrganization'] = getVOForGroup( '' )
try:
import gfalthr
infoDict['gfal version'] = gfalthr.gfal_version()
except:
pass
try:
import lcg_util
infoDict['lcg_util version'] = lcg_util.lcg_util_version()
except:
pass
def getGroupedPilotSummary(self, selectDict, columnList):
"""
The simplified pilot summary based on getPilotSummaryWeb method. It calculates pilot efficiency
based on the same algorithm as in the Web version, basically takes into account Done and
Aborted pilots only from the last day. The selection is done entirely in SQL.
:param dict selectDict: A dictionary to pass additional conditions to select statements, i.e.
it allows to define start time for Done and Aborted Pilots. Unused.
:param list columnList: A list of column to consider when grouping to calculate efficiencies.
e.g. ['GridSite', 'DestinationSite'] is used to calculate efficiencies
for sites and CEs. If we want to add an OwnerGroup it would be:
['GridSite', 'DestinationSite', 'OwnerGroup'].
:return: S_OK/S_ERROR with a dict containing the ParameterNames and Records lists.
"""
# TODO:
# add startItem and maxItems to the argument list
# limit output to finalDict['Records'] = records[startItem:startItem + maxItems]
table = PivotedPilotSummaryTable(columnList)
sqlQuery = table.buildSQL()
self.log.debug("SQL query : ")
self.log.debug("\n" + sqlQuery)
res = self._query(sqlQuery)
if not res["OK"]:
return res
rows = []
columns = table.getColumnList()
try:
groupIndex = columns.index("OwnerGroup")
# should probably change a column name to VO here as well to avoid confusion
except ValueError:
groupIndex = None
result = {"ParameterNames": columns}
multiple = False
# If not grouped by CE:
if "CE" not in columns:
multiple = True
for row in res["Value"]:
lrow = list(row)
if groupIndex:
lrow[groupIndex] = getVOForGroup(row[groupIndex])
if multiple:
lrow.append("Multiple")
for index, value in enumerate(row):
if isinstance(value, decimal.Decimal):
lrow[index] = float(value)
# get the value of the Total column
if "Total" in columnList:
total = lrow[columnList.index("Total")]
else:
total = 0
if "PilotJobEff" in columnList:
eff = lrow[columnList.index("PilotJobEff")]
else:
eff = 0.0
lrow.append(self._getElementStatus(total, eff))
rows.append(list(lrow))
# If not grouped by CE and more then 1 CE in the result:
if multiple:
columns.append("CE") # 'DestinationSite' re-mapped to 'CE' already
columns.append("Status")
result["Records"] = rows
result["TotalRecords"] = len(rows)
return S_OK(result)
def _checkFilesToStage(seToLFNs,
onlineLFNs,
offlineLFNs,
absentLFNs,
checkOnlyTapeSEs=None,
jobLog=None,
proxyUserName=None,
proxyUserGroup=None,
executionLock=None):
"""
Checks on SEs whether the file is NEARLINE or ONLINE
onlineLFNs, offlineLFNs and absentLFNs are modified to contain the files found online
If checkOnlyTapeSEs is True, disk replicas are not checked
As soon as a replica is found Online for a file, no further check is made
"""
# Only check on storage if it is a tape SE
if jobLog is None:
logger = gLogger
else:
logger = jobLog
if checkOnlyTapeSEs is None:
# Default value is True
checkOnlyTapeSEs = True
failed = {}
for se, lfnsInSEList in seToLFNs.items():
# If we have found already all files online at another SE, no need to check the others
# but still we want to set the SE as Online if not a TapeSE
vo = getVOForGroup(proxyUserGroup)
seObj = StorageElement(se, vo=vo)
status = seObj.getStatus()
if not status['OK']:
return status
tapeSE = status['Value']['TapeSE']
diskSE = status['Value']['DiskSE']
# If requested to check only Tape SEs and the file is at a diskSE, we guess it is Online...
filesToCheck = []
for lfn in lfnsInSEList:
# If the file had already been found accessible at an SE, only check that this one is on disk
diskIsOK = checkOnlyTapeSEs or (lfn in onlineLFNs)
if diskIsOK and diskSE:
onlineLFNs.setdefault(lfn, []).append(se)
elif not diskIsOK or (tapeSE and (lfn not in onlineLFNs)):
filesToCheck.append(lfn)
if not filesToCheck:
continue
# We have to use a new SE object because it caches the proxy!
with UserProxy(proxyUserName=proxyUserName,
proxyUserGroup=proxyUserGroup,
executionLock=executionLock) as proxyResult:
if proxyResult['OK']:
fileMetadata = StorageElement(
se, vo=vo).getFileMetadata(filesToCheck)
else:
fileMetadata = proxyResult
if not fileMetadata['OK']:
failed[se] = dict.fromkeys(filesToCheck, fileMetadata['Message'])
else:
if fileMetadata['Value']['Failed']:
failed[se] = fileMetadata['Value']['Failed']
# is there at least one replica online?
for lfn, mDict in fileMetadata['Value']['Successful'].items():
# SRM returns Cached, but others may only return Accessible
if mDict.get('Cached', mDict['Accessible']):
onlineLFNs.setdefault(lfn, []).append(se)
elif tapeSE:
# A file can be staged only at Tape SE
offlineLFNs.setdefault(lfn, []).append(se)
else:
# File not available at a diskSE... we shall retry later
pass
# Doesn't matter if some files are Offline if they are also online
for lfn in set(offlineLFNs) & set(onlineLFNs):
offlineLFNs.pop(lfn)
# If the file was found staged, ignore possible errors, but print out errors
for se, failedLfns in list(failed.items()):
logger.error("Errors when getting files metadata", 'at %s' % se)
for lfn, reason in list(failedLfns.items()):
if lfn in onlineLFNs:
logger.warn(reason,
'for %s, but there is an online replica' % lfn)
failed[se].pop(lfn)
else:
logger.error(reason, 'for %s, no online replicas' % lfn)
if cmpError(reason, errno.ENOENT):
absentLFNs.setdefault(lfn, []).append(se)
failed[se].pop(lfn)
if not failed[se]:
failed.pop(se)
# Find the files that do not exist at SE
if failed:
logger.error(
"Error getting metadata", "for %d files" %
len(set(lfn for lfnList in failed.values() for lfn in lfnList)))
for lfn in absentLFNs:
seList = absentLFNs[lfn]
# FIXME: it is not possible to return here an S_ERROR(), return the message only
absentLFNs[lfn] = S_ERROR(errno.ENOENT, "File not at %s" %
','.join(sorted(seList)))['Message']
# Format the error for absent files
return S_OK()
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1],
'Usage:',
' %s [option|cfgfile] ... Site' % Script.scriptName,
]))
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
infoDict = {}
infoDict['Setup'] = gConfig.getValue('/DIRAC/Setup', 'Unknown')
infoDict['ConfigurationServer'] = gConfig.getValue(
'/DIRAC/Configuration/Servers', [])
ret = getProxyInfo(disableVOMS=True)
if ret['OK'] and 'group' in ret['Value']:
infoDict['VirtualOrganization'] = getVOForGroup(ret['Value']['group'])
else:
infoDict['VirtualOrganization'] = getVOForGroup('')
try:
import gfalthr
infoDict['gfal version'] = gfalthr.gfal_version()
except:
pass
try:
import lcg_util
infoDict['lcg_util version'] = lcg_util.lcg_util_version()
except:
pass
'Arguments:',
' Site: Name of the Site (ie: CERN-PROD)'] ) )
Script.parseCommandLine( ignoreErrors = True )
args = Script.getPositionalArgs()
if not len( args ) == 1:
Script.showHelp()
site = args[0]
host = None
voName = None
ret = getProxyInfo( disableVOMS = True )
if ret['OK'] and 'group' in ret['Value']:
voName = getVOForGroup( ret['Value']['group'] )
for unprocSw in Script.getUnprocessedSwitches():
if unprocSw[0] in ( "H", "host" ):
host = unprocSw[1]
if unprocSw[0] in ( "V", "vo" ):
voName = unprocSw[1]
if not voName:
Script.gLogger.error( 'Could not determine VO' )
Script.showHelp()
from DIRAC import exit as DIRACExit
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
diracAdmin = DiracAdmin()
def web_getLaunchpadSetupWithLFNs(self):
#on the fly file catalog for advanced launchpad
if not hasattr(self, 'fc'):
userData = self.getSessionData()
group = str(userData["user"]["group"])
vo = getVOForGroup(group)
self.fc = FileCatalog(vo=vo)
self.set_header('Content-type', 'text/plain')
lfnList = []
arguments = self.request.arguments
gLogger.always(
"submit: incoming arguments %s to getLaunchpadSetupWithLFNs" %
arguments)
lfnStr = str(arguments['path'][0])
lfnList = lfnStr.split(',')
#checks if the experiments folder in lfn list has a rtg_def.m file at some subfolder
gLogger.always("submit: checking if some rtg_def.m" % arguments)
processed = []
metaDict = {'type': 'info'}
for lfn in lfnStr.split(','):
pos_relative = lfn.find("/")
pos_relative = lfn.find("/", pos_relative + 1)
pos_relative = lfn.find("/", pos_relative + 1)
pos_relative = lfn.find("/", pos_relative + 1)
pos_relative = lfn.find("/", pos_relative + 1)
experiment_lfn = lfn[0:pos_relative]
if experiment_lfn in processed:
continue
processed.append(experiment_lfn)
gLogger.always("checking rtg_def.m in %s" % experiment_lfn)
result = self.fc.findFilesByMetadata(metaDict,
path=str(experiment_lfn))
print "result"
print result
if not result['OK'] or not result['Value']:
gLogger.error("Failed to get type info from $s, %s" %
(experiment_lfn, result["Message"]))
continue
for candidate_lfn in result['Value']:
if candidate_lfn.find('rtg_def.m') > 0:
lfnList.append(candidate_lfn)
totalfn = len(lfnList)
ptlfn = ''
current = 1
for lfn in lfnList:
ptlfn = ptlfn + lfn
if current < totalfn:
ptlfn = ptlfn + ', '
current = current + 1
defaultParams = {
"JobName": [1, 'Eiscat'],
"Executable": [1, "/bin/ls"],
"Arguments": [1, "-ltrA"],
"OutputSandbox": [1, "std.out, std.err"],
"InputData": [1, ptlfn],
"OutputData": [0, ""],
"OutputSE": [1, "EISCAT-disk"],
"OutputPath": [0, ""],
"CPUTime": [0, "86400"],
"Site": [0, ""],
"BannedSite": [0, ""],
"Platform": [0, "Linux_x86_64_glibc-2.5"],
"Priority": [0, "5"],
"StdError": [0, "std.err"],
"StdOutput": [0, "std.out"],
"Parameters": [0, "0"],
"ParameterStart": [0, "0"],
"ParameterStep": [0, "1"]
}
delimiter = gConfig.getValue("/Website/Launchpad/ListSeparator", ',')
options = self.__getOptionsFromCS(delimiter=delimiter)
# platform = self.__getPlatform()
# if platform and options:
# if not options.has_key("Platform"):
# options[ "Platform" ] = platform
# else:
# csPlatform = list(options[ "Platform" ])
# allPlatforms = csPlatform + platform
# platform = uniqueElements(allPlatforms)
# options[ "Platform" ] = platform
gLogger.debug("Options from CS: %s" % options)
override = gConfig.getValue("/Website/Launchpad/OptionsOverride",
False)
gLogger.info("end __getLaunchpadOpts")
# Updating the default values from OptionsOverride configuration branch,
for key in options:
if key not in defaultParams:
defaultParams[key] = [0, ""]
defaultParams[key][1] = options[key][0]
gLogger.info(
"Default params + override from /Website/Launchpad/OptionsOverride -> %s"
% defaultParams)
# Reading of the predefined sets of launchpad parameters values
obj = Operations()
predefinedSets = {}
launchpadSections = obj.getSections("Launchpad")
import pprint
if launchpadSections['OK']:
for section in launchpadSections["Value"]:
predefinedSets[section] = {}
sectionOptions = obj.getOptionsDict("Launchpad/" + section)
pprint.pprint(sectionOptions)
if sectionOptions['OK']:
predefinedSets[section] = sectionOptions["Value"]
self.write({
"success": "true",
"result": defaultParams,
"predefinedSets": predefinedSets
})
if years:
totalDays += 365 * years
if months:
totalDays += 30 * months
if days:
totalDays += days
res = getProxyInfo(False, False)
if not res['OK']:
gLogger.error("Failed to get client proxy information.", res['Message'])
DIRAC.exit(2)
proxyInfo = res['Value']
username = proxyInfo['username']
vo = ''
if 'group' in proxyInfo:
vo = getVOForGroup(proxyInfo['group'])
if not baseDir:
if not vo:
gLogger.error('Could not determine VO')
Script.showHelp()
baseDir = '/%s/user/%s/%s' % (vo, username[0], username)
baseDir = baseDir.rstrip('/')
gLogger.info('Will search for files in %s' % baseDir)
activeDirs = [baseDir]
allFiles = []
allDirs = []
emptyDirs = []
while len(activeDirs) > 0:
)
)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if not len(args) == 1:
Script.showHelp()
site = args[0]
host = None
voName = None
ret = getProxyInfo(disableVOMS=True)
if ret["OK"] and "group" in ret["Value"]:
voName = getVOForGroup(ret["Value"]["group"])
for unprocSw in Script.getUnprocessedSwitches():
if unprocSw[0] in ("H", "host"):
host = unprocSw[1]
if unprocSw[0] in ("V", "vo"):
voName = unprocSw[1]
if not voName:
Script.gLogger.error("Could not determine VO")
Script.showHelp()
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
diracAdmin = DiracAdmin()
def main():
import os
import DIRAC
from DIRAC import gConfig
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup
from DIRAC.Core.Utilities.PrettyPrint import printTable
def version(arg):
Script.disableCS()
print(DIRAC.version)
DIRAC.exit(0)
def platform(arg):
Script.disableCS()
print(DIRAC.getPlatform())
DIRAC.exit(0)
Script.registerSwitch("v", "version",
"print version of current DIRAC installation",
version)
Script.registerSwitch("p", "platform",
"print platform of current DIRAC installation",
platform)
Script.parseCommandLine(ignoreErrors=True)
records = []
records.append(("Setup", gConfig.getValue("/DIRAC/Setup", "Unknown")))
records.append((
"AuthorizationServer",
gConfig.getValue(
"/DIRAC/Security/Authorization/issuer",
"/DIRAC/Security/Authorization/issuer option is absent"),
))
records.append(("ConfigurationServer",
gConfig.getValue("/DIRAC/Configuration/Servers",
"None found")))
records.append(("Installation path", DIRAC.rootPath))
if os.path.exists(
os.path.join(DIRAC.rootPath, DIRAC.getPlatform(), "bin", "mysql")):
records.append(("Installation type", "server"))
else:
records.append(("Installation type", "client"))
records.append(("Platform", DIRAC.getPlatform()))
ret = getProxyInfo(disableVOMS=True)
if ret["OK"]:
if "group" in ret["Value"]:
vo = getVOForGroup(ret["Value"]["group"])
else:
vo = getVOForGroup("")
if not vo:
vo = "None"
records.append(("VirtualOrganization", vo))
if "identity" in ret["Value"]:
records.append(("User DN", ret["Value"]["identity"]))
if "secondsLeft" in ret["Value"]:
records.append(("Proxy validity, secs", {
"Value": str(ret["Value"]["secondsLeft"]),
"Just": "L"
}))
if gConfig.getValue("/DIRAC/Security/UseServerCertificate", True):
records.append(("Use Server Certificate", "Yes"))
else:
records.append(("Use Server Certificate", "No"))
if gConfig.getValue("/DIRAC/Security/UseTokens",
"false").lower() in ("y", "yes", "true"):
records.append(("Use tokens", "Yes"))
else:
records.append(("Use tokens", "No"))
if gConfig.getValue("/DIRAC/Security/SkipCAChecks", False):
records.append(("Skip CA Checks", "Yes"))
else:
records.append(("Skip CA Checks", "No"))
records.append(("DIRAC version", DIRAC.version))
fields = ["Option", "Value"]
print()
printTable(fields, records, numbering=False)
print()
records.append( ('Setup', gConfig.getValue( '/DIRAC/Setup', 'Unknown' ) ) )
records.append( ('ConfigurationServer', str( gConfig.getValue( '/DIRAC/Configuration/Servers', [] ) ) ) )
records.append( ('Installation path', DIRAC.rootPath ) )
if os.path.exists( os.path.join( DIRAC.rootPath, DIRAC.getPlatform(), 'bin', 'mysql' ) ):
records.append( ('Installation type', 'server' ) )
else:
records.append( ('Installation type', 'client' ) )
records.append( ( 'Platform', DIRAC.getPlatform() ) )
ret = getProxyInfo( disableVOMS = True )
if ret['OK']:
if 'group' in ret['Value']:
records.append( ('VirtualOrganization', getVOForGroup( ret['Value']['group'] ) ) )
else:
records.append( ('VirtualOrganization', getVOForGroup( '' ) ) )
if 'identity' in ret['Value']:
records.append( ('User DN', ret['Value']['identity'] ) )
if 'secondsLeft' in ret['Value']:
records.append( ('Proxy validity, secs', str( ret['Value']['secondsLeft'] ) ) )
if gConfig.getValue( '/DIRAC/Security/UseServerCertificate', True ):
records.append( ('Use Server Certificate', 'Yes' ) )
else:
records.append( ('Use Server Certificate', 'No' ) )
if gConfig.getValue( '/DIRAC/Security/SkipCAChecks', False ):
records.append( ('Skip CA Checks', 'Yes' ) )
else:
records.append( ('Skip CA Checks', 'No' ) )
