def __init__( self, *args, **kwargs ):
super( DiracWebApp, self ).__init__( *args, **kwargs )
#Find base modules path
self.__baseControllerPaths = []
for module in getCSExtensions():
module = "%sDIRAC" % module
modulePath = os.path.join( rootPath, module )
if os.path.isdir( os.path.join( modulePath, "Web", "controllers" ) ):
self.__baseControllerPaths.append( "%s.Web.controllers" % module )
for systemName in os.listdir( modulePath ):
if os.path.isdir( os.path.join( modulePath, systemName, "Web", "controllers" ) ):
self.__baseControllerPaths.append( "%s.%s.Web.controllers" % ( module, systemName ) )
self.__baseControllerPaths.append( 'dirac.controllers' )
self.__log = gLogger.getSubLogger( "DIRACWebApp" )
self.__log.info( "Base modules to find controllers are:\n\t%s" % "\n\t".join ( self.__baseControllerPaths ) )
def portalVersion(rootPath):
from DIRAC.Core.Utilities import InstallTools
from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import getCSExtensions
result = InstallTools.getInfo(getCSExtensions())
if not result["OK"]:
return getRelease(rootPath)
version = result["Value"]
if not "Extensions" in version:
return getRelease(rootPath)
extensions = version["Extensions"]
if not "DIRACWeb" in extensions:
return getRelease(rootPath)
return extensions["DIRACWeb"]
def portalVersion( rootPath ):
from DIRAC.Core.Utilities import InstallTools
from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import getCSExtensions
result = InstallTools.getInfo( getCSExtensions() )
if not result[ "OK" ]:
return getRelease( rootPath )
version = result[ "Value" ]
if not "Extensions" in version:
return getRelease( rootPath )
extensions = version[ "Extensions" ]
if not "DIRACWeb" in extensions:
return getRelease( rootPath )
return extensions[ "DIRACWeb" ]
def getCurrentVersion():
""" Get a string corresponding to the current version of the DIRAC package and all the installed
extension packages
"""
version = 'DIRAC ' + DIRAC.version
for ext in getCSExtensions():
try:
import imp
module = imp.find_module("%sDIRAC" % ext)
extModule = imp.load_module("%sDIRAC" % ext, *module)
version = extModule.version
except ImportError:
pass
except AttributeError:
pass
return S_OK(version)
def getCurrentVersion():
""" Get a string corresponding to the current version of the DIRAC package and all the installed
extension packages
"""
version = 'DIRAC ' + DIRAC.version
for ext in getCSExtensions():
try:
import imp
module = imp.find_module( "%sDIRAC" % ext )
extModule = imp.load_module( "%sDIRAC" % ext, *module )
version = extModule.version
except ImportError:
pass
except AttributeError:
pass
return S_OK( version )
def getVersion():
""" Get a dictionary corresponding to the current version of the DIRAC package and all the installed
extension packages
"""
vDict = {'Extensions': {}}
vDict['DIRAC'] = DIRAC.version
for ext in getCSExtensions():
try:
import imp
module = imp.find_module("%sDIRAC" % ext)
extModule = imp.load_module("%sDIRAC" % ext, *module)
vDict['Extensions'][ext] = extModule.version
except ImportError:
pass
except AttributeError:
pass
return S_OK(vDict)
def getVersion():
""" Get a dictionary corresponding to the current version of the DIRAC package and all the installed
extension packages
"""
vDict = {'Extensions':{}}
vDict['DIRAC'] = DIRAC.version
for ext in getCSExtensions():
try:
import imp
module = imp.find_module( "%sDIRAC" % ext )
extModule = imp.load_module( "%sDIRAC" % ext, *module )
vDict['Extensions'][ext] = extModule.version
except ImportError:
pass
except AttributeError:
pass
return S_OK( vDict )
Script.parseCommandLine()
args = Script.getPositionalArgs()
if len( args ) == 1:
args = args[0].split( '/' )
if len( args ) != 2:
Script.showHelp()
DIRACexit( -1 )
#
system = args[0]
executor = args[1]
if module:
result = InstallTools.addDefaultOptionsToCS( gConfig, 'executor', system, module,
getCSExtensions(),
overwrite = overwrite )
result = InstallTools.addDefaultOptionsToCS( gConfig, 'executor', system, executor,
getCSExtensions(),
specialOptions=specialOptions,
overwrite = overwrite,
addDefaultOptions = False )
else:
result = InstallTools.addDefaultOptionsToCS( gConfig, 'executor', system, executor,
getCSExtensions(),
specialOptions=specialOptions,
overwrite = overwrite )
if not result['OK']:
print "ERROR:", result['Message']
else:
result = InstallTools.installComponent( 'executor', system, executor, getCSExtensions(), module )
def main():
global overwrite
global specialOptions
global module
global specialOptions
from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller
gComponentInstaller.exitOnError = True
Script.registerSwitch("w", "overwrite",
"Overwrite the configuration in the global CS",
setOverwrite)
Script.registerSwitch("m:", "module=",
"Python module name for the component code",
setModule)
Script.registerSwitch("p:", "parameter=", "Special component option ",
setSpecialOption)
Script.parseCommandLine()
args = Script.getPositionalArgs()
if len(args) == 1:
args = args[0].split('/')
if len(args) != 2:
Script.showHelp()
DIRACexit(1)
system = args[0]
component = args[1]
compOrMod = module if module else component
result = gComponentInstaller.addDefaultOptionsToCS(
gConfig,
'service',
system,
component,
getCSExtensions(),
specialOptions=specialOptions,
overwrite=overwrite)
if not result['OK']:
gLogger.error(result['Message'])
DIRACexit(1)
result = gComponentInstaller.addTornadoOptionsToCS(gConfig)
if not result['OK']:
gLogger.error(result['Message'])
DIRACexit(1)
result = gComponentInstaller.installTornado()
if not result['OK']:
gLogger.error(result['Message'])
DIRACexit(1)
gLogger.notice(
'Successfully installed component %s in %s system, now setting it up' %
(component, system))
result = gComponentInstaller.setupTornadoService(system, component,
getCSExtensions(), module)
if not result['OK']:
gLogger.error(result['Message'])
DIRACexit(1)
result = MonitoringUtilities.monitorInstallation('service', system,
component, module)
if not result['OK']:
gLogger.error(result['Message'])
DIRACexit(1)
gLogger.notice('Successfully completed the installation of %s/%s' %
(system, component))
DIRACexit()
Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
'Usage:',
' %s [option|cfgfile] ... System Service|System/Service' % Script.scriptName,
'Arguments:',
' System: Name of the DIRAC system (ie: WorkloadManagement)',
' Service: Name of the DIRAC service (ie: Matcher)'] ) )
Script.parseCommandLine()
args = Script.getPositionalArgs()
if len( args ) == 1:
args = args[0].split( '/' )
if len( args ) != 2:
Script.showHelp()
exit( -1 )
#
system = args[0]
service = args[1]
result = InstallTools.addDefaultOptionsToCS( gConfig, 'service', system, service,
getCSExtensions(), overwrite = overwrite )
if not result['OK']:
print "ERROR:", result['Message']
else:
result = InstallTools.installComponent( 'service', system, service, getCSExtensions() )
if not result['OK']:
print "ERROR:", result['Message']
else:
print "Successfully installed service %s in %s system" % ( service, system )
def initDIRAC( rootPath, enableDebug = False ):
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
configDict = { 'webConfig' : {} }
configDict[ 'webConfig' ]['dirac.webroot'] = rootPath
diracRootPath = os.path.realpath( os.path.dirname( os.path.dirname( rootPath ) ) )
configDict[ 'webConfig' ]['dirac.root'] = diracRootPath
if diracRootPath not in sys.path:
sys.path.append( diracRootPath )
from DIRAC.FrameworkSystem.Client.Logger import gLogger
gLogger.registerBackends( [ 'stderr' ] )
from DIRAC.Core.Base import Script
Script.registerSwitch( "r", "reload", "Reload for pylons" )
Script.localCfg.addDefaultEntry( "/DIRAC/Security/UseServerCertificate", "yes" )
Script.localCfg.addDefaultEntry( "LogColor", True )
Script.initialize( script = "Website", ignoreErrors = True, initializeMonitor = False )
gLogger._systemName = "Framework"
gLogger.initialize( "Web", "/Website" )
gLogger.setLevel( "VERBOSE" )
from DIRAC import gMonitor, gConfig, rootPath as droot
from DIRAC.Core.Utilities import CFG
from DIRAC.ConfigurationSystem.Client.Helpers import getCSExtensions
gMonitor.setComponentType( gMonitor.COMPONENT_WEB )
gMonitor.initialize()
gMonitor.registerActivity( "pagesServed", "Pages served", "Framework", "pages", gMonitor.OP_SUM )
gLogger.info( "DIRAC Initialized" )
configDict['portalVersion'] = portalVersion( rootPath )
gLogger.info( "DIRAC portal version: %s" % configDict['portalVersion'] )
extModules = [ '%sDIRAC' % module for module in getCSExtensions() ]
#Load web.cfg of modules
cfgFilePaths = [ os.path.join( droot, "etc", "web.cfg" ) ]
for extModule in extModules:
gLogger.info( "Adding web.cfg for %s extension" % extModule )
extModulePath = os.path.join( diracRootPath, extModule )
webCFGPath = os.path.join( extModulePath, "Web", "web.cfg" )
cfgFilePaths.append( webCFGPath )
for systemDir in os.listdir( extModulePath ):
webCFGSystemPath = os.path.join( extModulePath, systemDir, "Web", "web.cfg" )
cfgFilePaths.append( webCFGSystemPath )
webCFG = CFG.CFG()
for webCFGPath in cfgFilePaths:
if not os.path.isfile( webCFGPath ):
gLogger.warn( "%s does not exist" % webCFGPath )
else:
gLogger.info( "Loading %s" % webCFGPath )
modCFG = CFG.CFG().loadFromFile( webCFGPath )
if modCFG.getOption( 'Website/AbsoluteDefinition', False ):
gLogger.info( "CFG %s is absolute" % webCFGPath )
webCFG = modCFG
else:
webCFG = webCFG.mergeWith( modCFG )
gConfig.loadCFG( webCFG )
gLogger.showHeaders( True )
gLogger._gLogger__initialized = False
gLogger.initialize( "Web", "/Website" )
#Define the controllers, templates and public directories
for type in ( 'controllers', 'templates', 'public' ):
configDict[ type ] = []
for extModule in extModules:
extModulePath = os.path.join( diracRootPath, extModule )
typePath = os.path.join( extModulePath, "Web", type )
if os.path.isdir( typePath ):
gLogger.info( "Adding %s path for module %s" % ( type, extModule ) )
configDict[ type ].append( typePath )
for systemDir in os.listdir( extModulePath ):
systemTypePath = os.path.join( extModulePath, systemDir, "Web", type )
if os.path.isdir( systemTypePath ):
gLogger.info( "Adding %s path for system %s in module %s" % ( type, systemDir, extModule ) )
configDict[ type ].append( systemTypePath )
#End of extensions
configDict[ type ].append( os.path.join( rootPath, type ) )
#Load debug.cfg?
if enableDebug:
debugCFGPath = os.path.join( rootPath, "debug.cfg" )
if os.path.isfile( debugCFGPath ):
gLogger.info( "Loading debug cfg file at %s" % debugCFGPath )
gConfig.loadFile( debugCFGPath )
gLogger.info( "Extension modules loaded" )
return configDict
InstallTools.exitOnError = True
#
from DIRAC.Core.Base import Script
Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
'Usage:',
' %s [option|cfgfile] ... System Agent|System/Agent' % Script.scriptName,
'Arguments:',
' System: Name of the DIRAC system (ie: WorkloadManagement)',
' Agent: Name of the DIRAC agent (ie: JobCleaningAgent)'] ) )
Script.parseCommandLine()
args = Script.getPositionalArgs()
if len( args ) == 1:
args = args[0].split( '/' )
if len( args ) != 2:
Script.showHelp()
exit( -1 )
#
system = args[0]
agent = args[1]
result = InstallTools.addDefaultOptionsToCS( gConfig, 'agent', system, agent, getCSExtensions(), overwrite = True )
if not result['OK']:
print "ERROR:", result['Message']
else:
result = InstallTools.installComponent( 'agent', system, agent, getCSExtensions() )
if not result['OK']:
print "ERROR:", result['Message']
else:
print "Successfully installed agent %s in %s system" % ( agent, system )
def _getPilotOptions( self, taskQueueDict, pilotsToSubmit ):
# Need to limit the maximum number of pilots to submit at once
# For generic pilots this is limited by the number of use of the tokens and the
# maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation:
pilotsToSubmit = min( pilotsToSubmit, int( 50 / self.maxJobsInFillMode ) )
pilotOptions = []
privateIfGenericTQ = self.privatePilotFraction > random.random()
privateTQ = ( 'PilotTypes' in taskQueueDict and 'private' in [ t.lower() for t in taskQueueDict['PilotTypes'] ] )
forceGeneric = 'ForceGeneric' in taskQueueDict
submitPrivatePilot = ( privateIfGenericTQ or privateTQ ) and not forceGeneric
if submitPrivatePilot:
self.log.verbose( 'Submitting private pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] )
ownerDN = taskQueueDict['OwnerDN']
ownerGroup = taskQueueDict['OwnerGroup']
# User Group requirement
pilotOptions.append( '-G %s' % taskQueueDict['OwnerGroup'] )
# check if group allows jobsharing
ownerGroupProperties = getPropertiesForGroup( ownerGroup )
if not 'JobSharing' in ownerGroupProperties:
# Add Owner requirement to pilot
pilotOptions.append( "-O '%s'" % ownerDN )
if privateTQ:
pilotOptions.append( '-o /Resources/Computing/CEDefaults/PilotType=private' )
maxJobsInFillMode = self.maxJobsInFillMode
else:
#For generic jobs we'll submit mixture of generic and private pilots
self.log.verbose( 'Submitting generic pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] )
ownerDN = self.genericPilotDN
ownerGroup = self.genericPilotGroup
result = gProxyManager.requestToken( ownerDN, ownerGroup, max( pilotsToSubmit, self.maxJobsInFillMode ) )
if not result[ 'OK' ]:
self.log.error( ERROR_TOKEN, result['Message'] )
return S_ERROR( ERROR_TOKEN )
( token, numberOfUses ) = result[ 'Value' ]
pilotsToSubmit = min( numberOfUses, pilotsToSubmit )
pilotOptions.append( '-o /Security/ProxyToken=%s' % token )
pilotsToSubmit = ( pilotsToSubmit - 1 ) / self.maxJobsInFillMode + 1
maxJobsInFillMode = int( numberOfUses / pilotsToSubmit )
# Use Filling mode
pilotOptions.append( '-M %s' % maxJobsInFillMode )
# Debug
pilotOptions.append( '-d' )
# Setup.
pilotOptions.append( '-S %s' % taskQueueDict['Setup'] )
# CS Servers
csServers = gConfig.getValue( "/DIRAC/Configuration/Servers", [] )
pilotOptions.append( '-C %s' % ",".join( csServers ) )
# DIRAC Extensions
extensionsList = getCSExtensions()
if extensionsList:
pilotOptions.append( '-e %s' % ",".join( extensionsList ) )
# Requested version of DIRAC
pilotOptions.append( '-r %s' % self.installVersion )
# Requested Project to install
if self.installInstallation:
pilotOptions.append( '-V %s' % self.installInstallation )
# Requested CPU time
pilotOptions.append( '-T %s' % taskQueueDict['CPUTime'] )
if self.extraPilotOptions:
pilotOptions.extend( self.extraPilotOptions )
return S_OK( ( pilotOptions, pilotsToSubmit, ownerDN, ownerGroup, submitPrivatePilot, privateTQ ) )
def __getPilotOptions( self, queue, pilotsToSubmit ):
""" Prepare pilot options
"""
queueDict = self.queueDict[queue]['ParametersDict']
vo = getVO()
if not vo:
self.log.error( 'Virtual Organization is not defined in the configuration' )
return None
pilotOptions = [ "-V '%s'" % vo ]
setup = gConfig.getValue( "/DIRAC/Setup", "unknown" )
if setup == 'unknown':
self.log.error( 'Setup is not defined in the configuration' )
return None
pilotOptions.append( '-S %s' % setup )
diracVersion = gConfig.getValue( "/Operations/%s/%s/Versions/PilotVersion" % ( vo, setup ), "unknown" )
if diracVersion == 'unknown':
self.log.error( 'PilotVersion is not defined in the configuration' )
return None
pilotOptions.append( '-r %s' % diracVersion )
projectName = gConfig.getValue( "/Operations/%s/%s/Versions/PilotProject" % ( vo, setup ), "unknown" )
if projectName == 'unknown':
self.log.info( 'PilotProject is not defined in the configuration' )
else:
pilotOptions.append( '-l %s' % projectName )
ownerDN = self.genericPilotDN
ownerGroup = self.genericPilotGroup
result = gProxyManager.requestToken( ownerDN, ownerGroup, pilotsToSubmit * 5 )
if not result[ 'OK' ]:
self.log.error( ERROR_TOKEN, result['Message'] )
return S_ERROR( ERROR_TOKEN )
( token, numberOfUses ) = result[ 'Value' ]
pilotOptions.append( '-o /Security/ProxyToken=%s' % token )
# Use Filling mode
pilotOptions.append( '-M %s' % 5 )
# Debug
pilotOptions.append( '-d' )
# CS Servers
csServers = gConfig.getValue( "/DIRAC/Configuration/Servers", [] )
pilotOptions.append( '-C %s' % ",".join( csServers ) )
# DIRAC Extensions
extensionsList = getCSExtensions()
if extensionsList:
pilotOptions.append( '-e %s' % ",".join( extensionsList ) )
# Requested CPU time
pilotOptions.append( '-T %s' % queueDict['CPUTime'] )
# SiteName
pilotOptions.append( '-n %s' % queueDict['Site'] )
if 'ClientPlatform' in queueDict:
pilotOptions.append( "-p '%s'" % queueDict['ClientPlatform'] )
if 'SharedArea' in queueDict:
pilotOptions.append( "-o '/LocalSite/SharedArea=%s'" % queueDict['SharedArea'] )
if 'SI00' in queueDict:
factor = float( queueDict['SI00'] ) / 250.
pilotOptions.append( "-o '/LocalSite/CPUScalingFactor=%s'" % factor )
pilotOptions.append( "-o '/LocalSite/CPUNormalizationFactor=%s'" % factor )
else:
if 'CPUScalingFactor' in queueDict:
pilotOptions.append( "-o '/LocalSite/CPUScalingFactor=%s'" % queueDict['CPUScalingFactor'] )
if 'CPUNormalizationFactor' in queueDict:
pilotOptions.append( "-o '/LocalSite/CPUNormalizationFactor=%s'" % queueDict['CPUNormalizationFactor'] )
self.log.verbose( "pilotOptions: ", ' '.join( pilotOptions ) )
return pilotOptions
def __getPilotOptions(self, queue, pilotsToSubmit):
""" Prepare pilot options
"""
queueDict = self.queueDict[queue]['ParametersDict']
vo = getVO()
if not vo:
self.log.error(
'Virtual Organization is not defined in the configuration')
return None
pilotOptions = ["-V '%s'" % vo]
setup = gConfig.getValue("/DIRAC/Setup", "unknown")
if setup == 'unknown':
self.log.error('Setup is not defined in the configuration')
return None
pilotOptions.append('-S %s' % setup)
diracVersion = gConfig.getValue(
"/Operations/%s/%s/Versions/PilotVersion" % (vo, setup), "unknown")
if diracVersion == 'unknown':
self.log.error('PilotVersion is not defined in the configuration')
return None
pilotOptions.append('-r %s' % diracVersion)
projectName = gConfig.getValue(
"/Operations/%s/%s/Versions/PilotInstallation" % (vo, setup), "")
if projectName == '':
self.log.info('DIRAC installation will be installed by pilots')
else:
pilotOptions.append('-l %s' % projectName)
ownerDN = self.genericPilotDN
ownerGroup = self.genericPilotGroup
result = gProxyManager.requestToken(ownerDN, ownerGroup,
pilotsToSubmit * 5)
if not result['OK']:
self.log.error(ERROR_TOKEN, result['Message'])
return S_ERROR(ERROR_TOKEN)
(token, numberOfUses) = result['Value']
pilotOptions.append('-o /Security/ProxyToken=%s' % token)
# Use Filling mode
pilotOptions.append('-M %s' % 5)
# Debug
pilotOptions.append('-d')
# CS Servers
csServers = gConfig.getValue("/DIRAC/Configuration/Servers", [])
pilotOptions.append('-C %s' % ",".join(csServers))
# DIRAC Extensions
extensionsList = getCSExtensions()
if extensionsList:
pilotOptions.append('-e %s' % ",".join(extensionsList))
# Requested CPU time
pilotOptions.append('-T %s' % queueDict['CPUTime'])
# SiteName
pilotOptions.append('-n %s' % queueDict['Site'])
if 'ClientPlatform' in queueDict:
pilotOptions.append("-p '%s'" % queueDict['ClientPlatform'])
if 'SharedArea' in queueDict:
pilotOptions.append("-o '/LocalSite/SharedArea=%s'" %
queueDict['SharedArea'])
if 'SI00' in queueDict:
factor = float(queueDict['SI00']) / 250.
pilotOptions.append("-o '/LocalSite/CPUScalingFactor=%s'" % factor)
pilotOptions.append("-o '/LocalSite/CPUNormalizationFactor=%s'" %
factor)
else:
if 'CPUScalingFactor' in queueDict:
pilotOptions.append("-o '/LocalSite/CPUScalingFactor=%s'" %
queueDict['CPUScalingFactor'])
if 'CPUNormalizationFactor' in queueDict:
pilotOptions.append(
"-o '/LocalSite/CPUNormalizationFactor=%s'" %
queueDict['CPUNormalizationFactor'])
self.log.verbose("pilotOptions: ", ' '.join(pilotOptions))
return pilotOptions
Script.registerSwitch( "w", "overwrite", "Overwrite the configuration in the global CS", setOverwrite )
Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
'Usage:',
' %s [option|cfgfile] ... System Agent|System/Agent' % Script.scriptName,
'Arguments:',
' System: Name of the DIRAC system (ie: WorkloadManagement)',
' Agent: Name of the DIRAC agent (ie: JobCleaningAgent)'] ) )
Script.parseCommandLine()
args = Script.getPositionalArgs()
if len( args ) == 1:
args = args[0].split( '/' )
if len( args ) != 2:
Script.showHelp()
exit( -1 )
#
system = args[0]
agent = args[1]
result = InstallTools.addDefaultOptionsToCS( gConfig, 'agent', system, agent,
getCSExtensions(), overwrite = overwrite )
if not result['OK']:
print "ERROR:", result['Message']
else:
result = InstallTools.installComponent( 'agent', system, agent, getCSExtensions() )
if not result['OK']:
print "ERROR:", result['Message']
else:
print "Successfully installed agent %s in %s system" % ( agent, system )
def _getPilotOptions( self, taskQueueDict, pilotsToSubmit ):
# Need to limit the maximum number of pilots to submit at once
# For generic pilots this is limited by the number of use of the tokens and the
# maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation:
pilotsToSubmit = max( min( pilotsToSubmit, int( 50 / self.maxJobsInFillMode ) ), 1 )
pilotOptions = []
privateIfGenericTQ = self.privatePilotFraction > random.random()
privateTQ = ( 'PilotTypes' in taskQueueDict and 'private' in [ t.lower() for t in taskQueueDict['PilotTypes'] ] )
forceGeneric = 'ForceGeneric' in taskQueueDict
submitPrivatePilot = ( privateIfGenericTQ or privateTQ ) and not forceGeneric
if submitPrivatePilot:
self.log.verbose( 'Submitting private pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] )
ownerDN = taskQueueDict['OwnerDN']
ownerGroup = taskQueueDict['OwnerGroup']
# User Group requirement
pilotOptions.append( '-G %s' % taskQueueDict['OwnerGroup'] )
# check if group allows jobsharing
ownerGroupProperties = getPropertiesForGroup( ownerGroup )
if not 'JobSharing' in ownerGroupProperties:
# Add Owner requirement to pilot
pilotOptions.append( "-O '%s'" % ownerDN )
if privateTQ:
pilotOptions.append( '-o /Resources/Computing/CEDefaults/PilotType=private' )
maxJobsInFillMode = self.maxJobsInFillMode
else:
#For generic jobs we'll submit mixture of generic and private pilots
self.log.verbose( 'Submitting generic pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] )
#ADRI: Find the generic group
result = findGenericPilotCredentials( group = taskQueueDict[ 'OwnerGroup' ] )
if not result[ 'OK' ]:
self.log.error( ERROR_GENERIC_CREDENTIALS, result[ 'Message' ] )
return S_ERROR( ERROR_GENERIC_CREDENTIALS )
ownerDN, ownerGroup = result[ 'Value' ]
result = gProxyManager.requestToken( ownerDN, ownerGroup, max( pilotsToSubmit, self.maxJobsInFillMode ) )
if not result[ 'OK' ]:
self.log.error( ERROR_TOKEN, result['Message'] )
return S_ERROR( ERROR_TOKEN )
( token, numberOfUses ) = result[ 'Value' ]
pilotsToSubmit = min( numberOfUses, pilotsToSubmit )
pilotOptions.append( '-o /Security/ProxyToken=%s' % token )
pilotsToSubmit = max( 1, ( pilotsToSubmit - 1 ) / self.maxJobsInFillMode + 1 )
maxJobsInFillMode = int( numberOfUses / pilotsToSubmit )
# Use Filling mode
pilotOptions.append( '-M %s' % maxJobsInFillMode )
# Debug
pilotOptions.append( '-d' )
# Setup.
pilotOptions.append( '-S %s' % taskQueueDict['Setup'] )
# CS Servers
csServers = gConfig.getServersList()
if len( csServers ) > 3:
# Remove the master
master = gConfigurationData.getMasterServer()
if master in csServers:
csServers.remove( master )
pilotOptions.append( '-C %s' % ",".join( csServers ) )
# DIRAC Extensions to be used in pilots
# ubeda: I'm not entirely sure if we can use here the same opsHelper as in line
# line +352
pilotExtensionsList = Operations().getValue( "Pilot/Extensions", [] )
extensionsList = []
if pilotExtensionsList:
if pilotExtensionsList[0] != 'None':
extensionsList = pilotExtensionsList
else:
extensionsList = getCSExtensions()
if extensionsList:
pilotOptions.append( '-e %s' % ",".join( extensionsList ) )
#Get DIRAC version and project, There might be global Setup defaults and per VO/Setup defaults (from configure)
opsHelper = Operations( group = taskQueueDict['OwnerGroup'], setup = taskQueueDict['Setup'] )
# Requested version of DIRAC (it can be a list, so we take the fist one)
version = opsHelper.getValue( cfgPath( 'Pilot', 'Version' ) , [ self.installVersion ] )[0]
pilotOptions.append( '-r %s' % version )
# Requested Project to install
installProject = opsHelper.getValue( cfgPath( 'Pilot', 'Project' ) , self.installProject )
if installProject:
pilotOptions.append( '-l %s' % installProject )
installation = opsHelper.getValue( cfgPath( 'Pilot', 'Installation' ), self.installation )
if installation:
pilotOptions.append( "-V %s" % installation )
# Requested CPU time
pilotOptions.append( '-T %s' % taskQueueDict['CPUTime'] )
if self.submitPoolOption not in self.extraPilotOptions:
pilotOptions.append( self.submitPoolOption )
if self.extraPilotOptions:
pilotOptions.extend( self.extraPilotOptions )
return S_OK( ( pilotOptions, pilotsToSubmit, ownerDN, ownerGroup, submitPrivatePilot, privateTQ ) )
' Agent: Name of the DIRAC agent (ie: JobCleaningAgent)'] ) )
Script.parseCommandLine()
args = Script.getPositionalArgs()
if len( args ) == 1:
args = args[0].split( '/' )
if len( args ) != 2:
Script.showHelp()
DIRACexit( -1 )
#
system = args[0]
agent = args[1]
if module:
result = InstallTools.addDefaultOptionsToCS( gConfig, 'agent', system, module,
getCSExtensions(),
overwrite = overwrite )
result = InstallTools.addDefaultOptionsToCS( gConfig, 'agent', system, agent,
getCSExtensions(),
specialOptions=specialOptions,
overwrite = overwrite,
addDefaultOptions = False )
else:
result = InstallTools.addDefaultOptionsToCS( gConfig, 'agent', system, agent,
getCSExtensions(),
specialOptions=specialOptions,
overwrite = overwrite )
if not result['OK']:
print "ERROR:", result['Message']
else:
result = InstallTools.installComponent( 'agent', system, agent, getCSExtensions(), module )
def do_install( self, args ):
"""
Install various DIRAC components
usage:
install mysql
install db <database>
install service <system> <service> [-m <ModuleName>] [-p <Option>=<Value>] [-p <Option>=<Value>] ...
install agent <system> <agent> [-m <ModuleName>] [-p <Option>=<Value>] [-p <Option>=<Value>] ...
install executor <system> <executor> [-m <ModuleName>] [-p <Option>=<Value>] [-p <Option>=<Value>] ...
"""
argss = args.split()
if not argss:
gLogger.notice( self.do_install.__doc__ )
return
option = argss[0]
del argss[0]
if option == "mysql":
gLogger.notice( "Installing MySQL database, this can take a while ..." )
client = SystemAdministratorClient( self.host, self.port )
if gComponentInstaller.mysqlPassword == 'LocalConfig':
gComponentInstaller.mysqlPassword = ''
gComponentInstaller.getMySQLPasswords()
result = client.installMySQL( gComponentInstaller.mysqlRootPwd, gComponentInstaller.mysqlPassword )
if not result['OK']:
self._errMsg( result['Message'] )
else:
gLogger.notice( "MySQL:", result['Value'] )
gLogger.notice( "You might need to restart SystemAdministrator service to take new settings into account" )
elif option == "db":
if not argss:
gLogger.notice( self.do_install.__doc__ )
return
database = argss[0]
client = SystemAdministratorClient( self.host, self.port )
result = client.getAvailableDatabases()
if not result['OK']:
self._errMsg( "Can not get database list: %s" % result['Message'] )
return
if not result['Value'].has_key( database ):
self._errMsg( "Unknown database %s: " % database )
return
system = result['Value'][database]['System']
setup = gConfig.getValue( '/DIRAC/Setup', '' )
if not setup:
self._errMsg( "Unknown current setup" )
return
instance = gConfig.getValue( '/DIRAC/Setups/%s/%s' % ( setup, system ), '' )
if not instance:
self._errMsg( "No instance defined for system %s" % system )
self._errMsg( "\tAdd new instance with 'add instance %s <instance_name>'" % system )
return
if not gComponentInstaller.mysqlPassword:
gComponentInstaller.mysqlPassword = '******'
gComponentInstaller.getMySQLPasswords()
result = client.installDatabase( database, gComponentInstaller.mysqlRootPwd )
if not result['OK']:
self._errMsg( result['Message'] )
return
extension, system = result['Value']
result = client.getHostInfo()
if not result[ 'OK' ]:
self._errMsg( result[ 'Message' ] )
return
else:
cpu = result[ 'Value' ][ 'CPUModel' ]
hostname = self.host
if not result[ 'OK' ]:
self._errMsg( result[ 'Message' ] )
return
if database != 'InstalledComponentsDB':
result = MonitoringUtilities.monitorInstallation( 'DB', system.replace( 'System', '' ), database, cpu = cpu, hostname = hostname )
if not result['OK']:
self._errMsg( result['Message'] )
return
# result = client.addDatabaseOptionsToCS( system, database )
gComponentInstaller.mysqlHost = self.host
result = client.getInfo()
if not result['OK']:
self._errMsg( result['Message'] )
hostSetup = result['Value']['Setup']
result = gComponentInstaller.addDatabaseOptionsToCS( gConfig, system, database, hostSetup, overwrite = True )
if not result['OK']:
self._errMsg( result['Message'] )
return
gLogger.notice( "Database %s from %s/%s installed successfully" % ( database, extension, system ) )
elif option in self.runitComponents:
if len( argss ) < 2:
gLogger.notice( self.do_install.__doc__ )
return
system = argss[0]
del argss[0]
component = argss[0]
del argss[0]
specialOptions = {}
module = ''
for i in range(len(argss)):
if argss[i] == "-m":
specialOptions['Module'] = argss[i+1]
module = argss[i+1]
if argss[i] == "-p":
opt,value = argss[i+1].split('=')
specialOptions[opt] = value
if module == component:
module = ''
client = SystemAdministratorClient( self.host, self.port )
# First need to update the CS
# result = client.addDefaultOptionsToCS( option, system, component )
gComponentInstaller.host = self.host
result = client.getInfo()
if not result['OK']:
self._errMsg( result['Message'] )
return
hostSetup = result['Value']['Setup']
# Install Module section if not yet there
if module:
result = gComponentInstaller.addDefaultOptionsToCS( gConfig, option, system, module,
getCSExtensions(), hostSetup )
# in case of Error we must stop, this can happen when the module name is wrong...
if not result['OK']:
self._errMsg( result['Message'] )
return
# Add component section with specific parameters only
result = gComponentInstaller.addDefaultOptionsToCS( gConfig, option, system, component,
getCSExtensions(), hostSetup, specialOptions,
addDefaultOptions = True )
else:
# Install component section
result = gComponentInstaller.addDefaultOptionsToCS( gConfig, option, system, component,
getCSExtensions(), hostSetup, specialOptions )
if not result['OK']:
self._errMsg( result['Message'] )
return
# Then we can install and start the component
result = client.setupComponent( option, system, component, module )
if not result['OK']:
self._errMsg( result['Message'] )
return
compType = result['Value']['ComponentType']
runit = result['Value']['RunitStatus']
gLogger.notice( "%s %s_%s is installed, runit status: %s" % ( compType, system, component, runit ) )
# And register it in the database
result = client.getHostInfo()
if not result[ 'OK' ]:
self._errMsg( result[ 'Message' ] )
return
else:
cpu = result[ 'Value' ][ 'CPUModel' ]
hostname = self.host
if component == 'ComponentMonitoring':
# Make sure that the service is running before trying to use it
nTries = 0
maxTries = 5
mClient = ComponentMonitoringClient()
result = mClient.ping()
while not result[ 'OK' ] and nTries < maxTries:
time.sleep( 3 )
result = mClient.ping()
nTries = nTries + 1
if not result[ 'OK' ]:
self._errMsg( 'ComponentMonitoring service taking too long to start. Installation will not be logged into the database' )
return
result = MonitoringUtilities.monitorInstallation( 'DB', system, 'InstalledComponentsDB', cpu = cpu, hostname = hostname )
if not result['OK']:
self._errMsg( 'Error registering installation into database: %s' % result[ 'Message' ] )
return
result = MonitoringUtilities.monitorInstallation( option, system, component, module, cpu = cpu, hostname = hostname )
if not result['OK']:
self._errMsg( 'Error registering installation into database: %s' % result[ 'Message' ] )
return
else:
gLogger.notice( "Unknown option:", option )
def _getPilotOptions(self, taskQueueDict, pilotsToSubmit):
# Need to limit the maximum number of pilots to submit at once
# For generic pilots this is limited by the number of use of the tokens and the
# maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation:
pilotsToSubmit = max(min(pilotsToSubmit, int(50 / self.maxJobsInFillMode)), 1)
pilotOptions = []
privateIfGenericTQ = self.privatePilotFraction > random.random()
privateTQ = "PilotTypes" in taskQueueDict and "private" in [t.lower() for t in taskQueueDict["PilotTypes"]]
forceGeneric = "ForceGeneric" in taskQueueDict
submitPrivatePilot = (privateIfGenericTQ or privateTQ) and not forceGeneric
if submitPrivatePilot:
self.log.verbose("Submitting private pilots for TaskQueue %s" % taskQueueDict["TaskQueueID"])
ownerDN = taskQueueDict["OwnerDN"]
ownerGroup = taskQueueDict["OwnerGroup"]
# User Group requirement
pilotOptions.append("-G %s" % taskQueueDict["OwnerGroup"])
# check if group allows jobsharing
ownerGroupProperties = getPropertiesForGroup(ownerGroup)
if not "JobSharing" in ownerGroupProperties:
# Add Owner requirement to pilot
pilotOptions.append("-O '%s'" % ownerDN)
if privateTQ:
pilotOptions.append("-o /Resources/Computing/CEDefaults/PilotType=private")
maxJobsInFillMode = self.maxJobsInFillMode
else:
# For generic jobs we'll submit mixture of generic and private pilots
self.log.verbose("Submitting generic pilots for TaskQueue %s" % taskQueueDict["TaskQueueID"])
# ADRI: Find the generic group
result = findGenericPilotCredentials(group=taskQueueDict["OwnerGroup"])
if not result["OK"]:
self.log.error(ERROR_GENERIC_CREDENTIALS, result["Message"])
return S_ERROR(ERROR_GENERIC_CREDENTIALS)
ownerDN, ownerGroup = result["Value"]
result = gProxyManager.requestToken(ownerDN, ownerGroup, max(pilotsToSubmit, self.maxJobsInFillMode))
if not result["OK"]:
self.log.error(ERROR_TOKEN, result["Message"])
return S_ERROR(ERROR_TOKEN)
(token, numberOfUses) = result["Value"]
pilotsToSubmit = min(numberOfUses, pilotsToSubmit)
pilotOptions.append("-o /Security/ProxyToken=%s" % token)
pilotsToSubmit = max(1, (pilotsToSubmit - 1) / self.maxJobsInFillMode + 1)
maxJobsInFillMode = int(numberOfUses / pilotsToSubmit)
# Use Filling mode
pilotOptions.append("-M %s" % maxJobsInFillMode)
# Debug
pilotOptions.append("-d")
# Setup.
pilotOptions.append("-S %s" % taskQueueDict["Setup"])
# CS Servers
csServers = gConfig.getServersList()
if len(csServers) > 3:
# Remove the master
master = gConfigurationData.getMasterServer()
if master in csServers:
csServers.remove(master)
pilotOptions.append("-C %s" % ",".join(csServers))
# DIRAC Extensions
extensionsList = getCSExtensions()
if extensionsList:
pilotOptions.append("-e %s" % ",".join(extensionsList))
# Get DIRAC version and project, There might be global Setup defaults and per VO/Setup defaults (from configure)
opsHelper = Operations(group=taskQueueDict["OwnerGroup"], setup=taskQueueDict["Setup"])
# Requested version of DIRAC (it can be a list, so we take the fist one)
version = opsHelper.getValue(cfgPath("Pilot", "Version"), [self.installVersion])[0]
pilotOptions.append("-r %s" % version)
# Requested Project to install
installProject = opsHelper.getValue(cfgPath("Pilot", "Project"), self.installProject)
if installProject:
pilotOptions.append("-l %s" % installProject)
installation = opsHelper.getValue(cfgPath("Pilot", "Installation"), self.installation)
if installation:
pilotOptions.append("-V %s" % installation)
# Requested CPU time
pilotOptions.append("-T %s" % taskQueueDict["CPUTime"])
if self.submitPoolOption not in self.extraPilotOptions:
pilotOptions.append(self.submitPoolOption)
if self.extraPilotOptions:
pilotOptions.extend(self.extraPilotOptions)
return S_OK((pilotOptions, pilotsToSubmit, ownerDN, ownerGroup, submitPrivatePilot, privateTQ))
args = Script.getPositionalArgs()
if len(args) == 1:
args = args[0].split('/')
if len(args) != 2:
Script.showHelp()
DIRACexit(1)
cType = None
system = args[0]
component = args[1]
compOrMod = module if module else component
result = gComponentInstaller.getSoftwareComponents(getCSExtensions())
if not result['OK']:
gLogger.error(result['Message'])
DIRACexit(1)
else:
availableComponents = result['Value']
for compType in availableComponents:
if system in availableComponents[compType] and compOrMod in availableComponents[compType][system]:
cType = compType[:-1].lower()
break
if not cType:
gLogger.error('Component %s/%s is not available for installation' % (system, component))
DIRACexit(1)
def do_install( self, args ):
"""
Install various DIRAC components
usage:
install mysql
install db <database>
install service <system> <service>
install agent <system> <agent>
"""
argss = args.split()
if not argss:
print self.do_install.__doc__
return
option = argss[0]
del argss[0]
if option == "mysql":
print "Installing MySQL database, this can take a while ..."
client = SystemAdministratorClient( self.host, self.port )
if InstallTools.mysqlPassword == 'LocalConfig':
InstallTools.mysqlPassword = ''
InstallTools.getMySQLPasswords()
result = client.installMySQL( InstallTools.mysqlRootPwd, InstallTools.mysqlPassword )
if not result['OK']:
self.__errMsg( result['Message'] )
else:
print "MySQL:", result['Value']
print "You might need to restart SystemAdministrator service to take new settings into account"
elif option == "db":
if not argss:
print self.do_install.__doc__
return
database = argss[0]
client = SystemAdministratorClient( self.host, self.port )
result = client.getAvailableDatabases()
if not result['OK']:
self.__errMsg( "Can not get database list: %s" % result['Message'] )
return
if not result['Value'].has_key( database ):
self.__errMsg( "Unknown database %s: " % database )
return
system = result['Value'][database]['System']
setup = gConfig.getValue( '/DIRAC/Setup', '' )
if not setup:
self.__errMsg( "Unknown current setup" )
return
instance = gConfig.getValue( '/DIRAC/Setups/%s/%s' % ( setup, system ), '' )
if not instance:
self.__errMsg( "No instance defined for system %s" % system )
self.__errMsg( "\tAdd new instance with 'add instance %s <instance_name>'" % system )
return
if not InstallTools.mysqlPassword:
InstallTools.mysqlPassword = '******'
InstallTools.getMySQLPasswords()
result = client.installDatabase( database, InstallTools.mysqlRootPwd )
if not result['OK']:
self.__errMsg( result['Message'] )
return
extension, system = result['Value']
# result = client.addDatabaseOptionsToCS( system, database )
InstallTools.mysqlHost = self.host
result = client.getInfo()
if not result['OK']:
self.__errMsg( result['Message'] )
hostSetup = result['Value']['Setup']
result = InstallTools.addDatabaseOptionsToCS( gConfig, system, database, hostSetup )
if not result['OK']:
self.__errMsg( result['Message'] )
return
print "Database %s from %s/%s installed successfully" % ( database, extension, system )
elif option == "service" or option == "agent":
if len( argss ) < 2:
print self.do_install.__doc__
return
system = argss[0]
component = argss[1]
client = SystemAdministratorClient( self.host, self.port )
# First need to update the CS
# result = client.addDefaultOptionsToCS( option, system, component )
InstallTools.host = self.host
result = client.getInfo()
if not result['OK']:
self.__errMsg( result['Message'] )
return
hostSetup = result['Value']['Setup']
result = InstallTools.addDefaultOptionsToCS( gConfig, option, system, component, getCSExtensions(), hostSetup )
if not result['OK']:
self.__errMsg( result['Message'] )
return
# Then we can install and start the component
result = client.setupComponent( option, system, component )
if not result['OK']:
self.__errMsg( result['Message'] )
return
compType = result['Value']['ComponentType']
runit = result['Value']['RunitStatus']
print "%s %s_%s is installed, runit status: %s" % ( compType, system, component, runit )
else:
print "Unknown option:", option
Script.parseCommandLine()
args = Script.getPositionalArgs()
if len( args ) == 1:
args = args[0].split( '/' )
if len( args ) != 2:
Script.showHelp()
exit( -1 )
#
system = args[0]
service = args[1]
if module:
result = InstallTools.addDefaultOptionsToCS( gConfig, 'service', system, module,
getCSExtensions(),
overwrite = overwrite )
result = InstallTools.addDefaultOptionsToCS( gConfig, 'service', system, service,
getCSExtensions(),
specialOptions=specialOptions,
overwrite = overwrite,
addDefaultOptions = False )
else:
result = InstallTools.addDefaultOptionsToCS( gConfig, 'service', system, service,
getCSExtensions(),
specialOptions=specialOptions,
overwrite = overwrite )
if not result['OK']:
print "ERROR:", result['Message']
else:
result = InstallTools.installComponent( 'service', system, service, getCSExtensions(), module )
Script.parseCommandLine()
args = Script.getPositionalArgs()
if len( args ) == 1:
args = args[0].split( '/' )
if len( args ) != 2:
Script.showHelp()
DIRACexit( 1 )
cType = None
system = args[0]
component = args[1]
result = gComponentInstaller.getSoftwareComponents( getCSExtensions() )
if not result[ 'OK' ]:
gLogger.error( result[ 'Message' ] )
DIRACexit( 1 )
else:
availableComponents = result[ 'Value' ]
for compType in availableComponents:
if system in availableComponents[ compType ] and component in availableComponents[ compType ][ system ]:
cType = compType[:-1].lower()
break
if not cType:
gLogger.error( 'Component %s/%s is not available for installation' % ( system, component ) )
DIRACexit( 1 )
"\n".join(
[
__doc__.split("\n")[1],
"Usage:",
" %s [option|cfgfile] ... ComponentType System Component|System/Component" % Script.scriptName,
"Arguments:",
" ComponentType: Name of the ComponentType (ie: agent)",
" System: Name of the DIRAC system (ie: WorkloadManagement)",
" component: Name of the DIRAC component (ie: JobCleaningAgent)",
]
)
)
Script.parseCommandLine()
args = Script.getPositionalArgs()
componentType = args[0]
if len(args) == 2:
system, component = args[1].split("/")
else:
system = args[1]
component = args[2]
result = InstallTools.addDefaultOptionsToCS(
gConfig, componentType, system, component, getCSExtensions(), specialOptions={}, overwrite=False
)
if not result["OK"]:
print "ERROR:", result["Message"]
else:
DIRACexit()
Script.parseCommandLine()
args = Script.getPositionalArgs()
if len(args) == 1:
args = args.split('/')
if len(args) != 2:
Script.showHelp()
DIRACexit(1)
cType = None
system = args[0]
component = args[1]
result = InstallTools.getSoftwareComponents(getCSExtensions())
if not result['OK']:
gLogger.error(result['Message'])
DIRACexit(1)
else:
availableComponents = result['Value']
for compType in availableComponents:
if system in availableComponents[
compType] and component in availableComponents[compType][system]:
cType = compType[:-1].lower()
break
if not cType:
gLogger.error('Component %s/%s is not available for installation' %
(system, component))
' System: Name of the DIRAC system (ie: WorkloadManagement)',
' Agent: Name of the DIRAC agent (ie: JobCleaningAgent)'
]))
Script.parseCommandLine()
args = Script.getPositionalArgs()
if len(args) == 1:
args = args[0].split('/')
if len(args) != 2:
Script.showHelp()
exit(-1)
#
system = args[0]
agent = args[1]
result = InstallTools.addDefaultOptionsToCS(gConfig,
'agent',
system,
agent,
getCSExtensions(),
overwrite=overwrite)
if not result['OK']:
print "ERROR:", result['Message']
else:
result = InstallTools.installComponent('agent', system, agent,
getCSExtensions())
if not result['OK']:
print "ERROR:", result['Message']
else:
print "Successfully installed agent %s in %s system" % (agent, system)
def _getPilotOptions( self, taskQueueDict, pilotsToSubmit ):
# Need to limit the maximum number of pilots to submit at once
# For generic pilots this is limited by the number of use of the tokens and the
# maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation:
pilotsToSubmit = min( pilotsToSubmit, int( 50 / self.maxJobsInFillMode ) )
pilotOptions = [ "-V %s" % self.virtualOrganization ]
privateIfGenericTQ = self.privatePilotFraction > random.random()
privateTQ = ( 'PilotTypes' in taskQueueDict and 'private' in [ t.lower() for t in taskQueueDict['PilotTypes'] ] )
forceGeneric = 'ForceGeneric' in taskQueueDict
submitPrivatePilot = ( privateIfGenericTQ or privateTQ ) and not forceGeneric
if submitPrivatePilot:
self.log.verbose( 'Submitting private pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] )
ownerDN = taskQueueDict['OwnerDN']
ownerGroup = taskQueueDict['OwnerGroup']
# User Group requirement
pilotOptions.append( '-G %s' % taskQueueDict['OwnerGroup'] )
# check if group allows jobsharing
ownerGroupProperties = getPropertiesForGroup( ownerGroup )
if not 'JobSharing' in ownerGroupProperties:
# Add Owner requirement to pilot
pilotOptions.append( "-O '%s'" % ownerDN )
if privateTQ:
pilotOptions.append( '-o /Resources/Computing/CEDefaults/PilotType=private' )
maxJobsInFillMode = self.maxJobsInFillMode
else:
#For generic jobs we'll submit mixture of generic and private pilots
self.log.verbose( 'Submitting generic pilots for TaskQueue %s' % taskQueueDict['TaskQueueID'] )
ownerDN = self.genericPilotDN
ownerGroup = self.genericPilotGroup
result = gProxyManager.requestToken( ownerDN, ownerGroup, max( pilotsToSubmit, self.maxJobsInFillMode ) )
if not result[ 'OK' ]:
self.log.error( ERROR_TOKEN, result['Message'] )
return S_ERROR( ERROR_TOKEN )
( token, numberOfUses ) = result[ 'Value' ]
pilotsToSubmit = min( numberOfUses, pilotsToSubmit )
pilotOptions.append( '-o /Security/ProxyToken=%s' % token )
pilotsToSubmit = ( pilotsToSubmit - 1 ) / self.maxJobsInFillMode + 1
maxJobsInFillMode = int( numberOfUses / pilotsToSubmit )
# Use Filling mode
pilotOptions.append( '-M %s' % maxJobsInFillMode )
# Debug
pilotOptions.append( '-d' )
# Setup.
pilotOptions.append( '-S %s' % taskQueueDict['Setup'] )
# CS Servers
csServers = gConfig.getValue( "/DIRAC/Configuration/Servers", [] )
pilotOptions.append( '-C %s' % ",".join( csServers ) )
# DIRAC Extensions
extensionsList = getCSExtensions()
if extensionsList:
pilotOptions.append( '-e %s' % ",".join( extensionsList ) )
# Requested version of DIRAC
pilotOptions.append( '-r %s' % self.installVersion )
# Requested Project to install
pilotOptions.append( '-V %s' % self.installInstallation )
# Requested CPU time
pilotOptions.append( '-T %s' % taskQueueDict['CPUTime'] )
if self.extraPilotOptions:
pilotOptions.extend( self.extraPilotOptions )
return S_OK( ( pilotOptions, pilotsToSubmit, ownerDN, ownerGroup, submitPrivatePilot, privateTQ ) )
def do_install(self, args):
"""
Install various DIRAC components
usage:
install mysql
install db <database>
install service <system> <service>
install agent <system> <agent>
"""
argss = args.split()
if not argss:
print self.do_install.__doc__
return
option = argss[0]
del argss[0]
if option == "mysql":
print "Installing MySQL database, this can take a while ..."
client = SystemAdministratorClient(self.host, self.port)
if InstallTools.mysqlPassword == 'LocalConfig':
InstallTools.mysqlPassword = ''
InstallTools.getMySQLPasswords()
result = client.installMySQL(InstallTools.mysqlRootPwd,
InstallTools.mysqlPassword)
if not result['OK']:
self.__errMsg(result['Message'])
else:
print "MySQL:", result['Value']
print "You might need to restart SystemAdministrator service to take new settings into account"
elif option == "db":
if not argss:
print self.do_install.__doc__
return
database = argss[0]
client = SystemAdministratorClient(self.host, self.port)
result = client.getAvailableDatabases()
if not result['OK']:
self.__errMsg("Can not get database list: %s" %
result['Message'])
return
if not result['Value'].has_key(database):
self.__errMsg("Unknown database %s: " % database)
return
system = result['Value'][database]['System']
setup = gConfig.getValue('/DIRAC/Setup', '')
if not setup:
self.__errMsg("Unknown current setup")
return
instance = gConfig.getValue(
'/DIRAC/Setups/%s/%s' % (setup, system), '')
if not instance:
self.__errMsg("No instance defined for system %s" % system)
self.__errMsg(
"\tAdd new instance with 'add instance %s <instance_name>'"
% system)
return
if not InstallTools.mysqlPassword:
InstallTools.mysqlPassword = '******'
InstallTools.getMySQLPasswords()
result = client.installDatabase(database,
InstallTools.mysqlRootPwd)
if not result['OK']:
self.__errMsg(result['Message'])
return
extension, system = result['Value']
# result = client.addDatabaseOptionsToCS( system, database )
InstallTools.mysqlHost = self.host
result = client.getInfo()
if not result['OK']:
self.__errMsg(result['Message'])
hostSetup = result['Value']['Setup']
result = InstallTools.addDatabaseOptionsToCS(
gConfig, system, database, hostSetup)
if not result['OK']:
self.__errMsg(result['Message'])
return
print "Database %s from %s/%s installed successfully" % (
database, extension, system)
elif option == "service" or option == "agent":
if len(argss) < 2:
print self.do_install.__doc__
return
system = argss[0]
component = argss[1]
client = SystemAdministratorClient(self.host, self.port)
# First need to update the CS
# result = client.addDefaultOptionsToCS( option, system, component )
InstallTools.host = self.host
result = client.getInfo()
if not result['OK']:
self.__errMsg(result['Message'])
return
hostSetup = result['Value']['Setup']
result = InstallTools.addDefaultOptionsToCS(
gConfig, option, system, component, getCSExtensions(),
hostSetup)
if not result['OK']:
self.__errMsg(result['Message'])
return
# Then we can install and start the component
result = client.setupComponent(option, system, component)
if not result['OK']:
self.__errMsg(result['Message'])
return
compType = result['Value']['ComponentType']
runit = result['Value']['RunitStatus']
print "%s %s_%s is installed, runit status: %s" % (
compType, system, component, runit)
else:
print "Unknown option:", option
def do_install( self, args ):
"""
Install various DIRAC components
usage:
install mysql
install db <database>
install service <system> <service> [-m <ModuleName>] [-p <Option>=<Value>] [-p <Option>=<Value>] ...
install agent <system> <agent> [-m <ModuleName>] [-p <Option>=<Value>] [-p <Option>=<Value>] ...
install executor <system> <executor> [-m <ModuleName>] [-p <Option>=<Value>] [-p <Option>=<Value>] ...
"""
argss = args.split()
if not argss:
gLogger.notice( self.do_install.__doc__ )
return
option = argss[0]
del argss[0]
if option == "mysql":
gLogger.notice( "Installing MySQL database, this can take a while ..." )
client = SystemAdministratorClient( self.host, self.port )
if gComponentInstaller.mysqlPassword == 'LocalConfig':
gComponentInstaller.mysqlPassword = ''
gComponentInstaller.getMySQLPasswords()
result = client.installMySQL( gComponentInstaller.mysqlRootPwd, gComponentInstaller.mysqlPassword )
if not result['OK']:
self._errMsg( result['Message'] )
else:
gLogger.notice( "MySQL:", result['Value'] )
gLogger.notice( "You might need to restart SystemAdministrator service to take new settings into account" )
elif option == "db":
if not argss:
gLogger.notice( self.do_install.__doc__ )
return
database = argss[0]
client = SystemAdministratorClient( self.host, self.port )
result = client.getAvailableDatabases()
if not result['OK']:
self._errMsg( "Can not get database list: %s" % result['Message'] )
return
if not result['Value'].has_key( database ):
self._errMsg( "Unknown database %s: " % database )
return
system = result['Value'][database]['System']
setup = gConfig.getValue( '/DIRAC/Setup', '' )
if not setup:
self._errMsg( "Unknown current setup" )
return
instance = gConfig.getValue( '/DIRAC/Setups/%s/%s' % ( setup, system ), '' )
if not instance:
self._errMsg( "No instance defined for system %s" % system )
self._errMsg( "\tAdd new instance with 'add instance %s <instance_name>'" % system )
return
if not gComponentInstaller.mysqlPassword:
gComponentInstaller.mysqlPassword = '******'
gComponentInstaller.getMySQLPasswords()
result = client.installDatabase( database, gComponentInstaller.mysqlRootPwd )
if not result['OK']:
self._errMsg( result['Message'] )
return
extension, system = result['Value']
result = client.getHostInfo()
if not result[ 'OK' ]:
self._errMsg( result[ 'Message' ] )
return
else:
cpu = result[ 'Value' ][ 'CPUModel' ]
hostname = self.host
if not result[ 'OK' ]:
self._errMsg( result[ 'Message' ] )
return
if database != 'InstalledComponentsDB':
result = MonitoringUtilities.monitorInstallation( 'DB', system.replace( 'System', '' ), database, cpu = cpu, hostname = hostname )
if not result['OK']:
self._errMsg( result['Message'] )
return
# result = client.addDatabaseOptionsToCS( system, database )
gComponentInstaller.mysqlHost = self.host
result = client.getInfo()
if not result['OK']:
self._errMsg( result['Message'] )
hostSetup = result['Value']['Setup']
result = gComponentInstaller.addDatabaseOptionsToCS( gConfig, system, database, hostSetup, overwrite = True )
if not result['OK']:
self._errMsg( result['Message'] )
return
gLogger.notice( "Database %s from %s/%s installed successfully" % ( database, extension, system ) )
elif option in self.runitComponents:
if len( argss ) < 2:
gLogger.notice( self.do_install.__doc__ )
return
system = argss[0]
del argss[0]
component = argss[0]
del argss[0]
specialOptions = {}
module = ''
for i in range(len(argss)):
if argss[i] == "-m":
specialOptions['Module'] = argss[i+1]
module = argss[i+1]
if argss[i] == "-p":
opt,value = argss[i+1].split('=')
specialOptions[opt] = value
if module == component:
module = ''
client = SystemAdministratorClient( self.host, self.port )
# First need to update the CS
# result = client.addDefaultOptionsToCS( option, system, component )
gComponentInstaller.host = self.host
result = client.getInfo()
if not result['OK']:
self._errMsg( result['Message'] )
return
hostSetup = result['Value']['Setup']
# Install Module section if not yet there
if module:
result = gComponentInstaller.addDefaultOptionsToCS( gConfig, option, system, module,
getCSExtensions(), hostSetup )
# in case of Error we must stop, this can happen when the module name is wrong...
if not result['OK']:
self._errMsg( result['Message'] )
return
# Add component section with specific parameters only
result = gComponentInstaller.addDefaultOptionsToCS( gConfig, option, system, component,
getCSExtensions(), hostSetup, specialOptions,
addDefaultOptions = True )
else:
# Install component section
result = gComponentInstaller.addDefaultOptionsToCS( gConfig, option, system, component,
getCSExtensions(), hostSetup, specialOptions )
if not result['OK']:
self._errMsg( result['Message'] )
return
# Then we can install and start the component
result = client.setupComponent( option, system, component, module )
if not result['OK']:
self._errMsg( result['Message'] )
return
compType = result['Value']['ComponentType']
runit = result['Value']['RunitStatus']
gLogger.notice( "%s %s_%s is installed, runit status: %s" % ( compType, system, component, runit ) )
# And register it in the database
result = client.getHostInfo()
if not result[ 'OK' ]:
self._errMsg( result[ 'Message' ] )
return
else:
cpu = result[ 'Value' ][ 'CPUModel' ]
hostname = self.host
if component == 'ComponentMonitoring':
# Make sure that the service is running before trying to use it
nTries = 0
maxTries = 5
mClient = ComponentMonitoringClient()
result = mClient.ping()
while not result[ 'OK' ] and nTries < maxTries:
time.sleep( 3 )
result = mClient.ping()
nTries = nTries + 1
if not result[ 'OK' ]:
self._errMsg( 'ComponentMonitoring service taking too long to start. Installation will not be logged into the database' )
return
result = MonitoringUtilities.monitorInstallation( 'DB', system, 'InstalledComponentsDB', cpu = cpu, hostname = hostname )
if not result['OK']:
self._errMsg( 'Error registering installation into database: %s' % result[ 'Message' ] )
return
result = MonitoringUtilities.monitorInstallation( option, system, component, module, cpu = cpu, hostname = hostname )
if not result['OK']:
self._errMsg( 'Error registering installation into database: %s' % result[ 'Message' ] )
return
else:
gLogger.notice( "Unknown option:", option )
if len(args) == 1:
args = args[0].split('/')
if len(args) != 2:
Script.showHelp()
DIRACexit(-1)
#
system = args[0]
executor = args[1]
if module:
result = InstallTools.addDefaultOptionsToCS(gConfig,
'executor',
system,
module,
getCSExtensions(),
overwrite=overwrite)
result = InstallTools.addDefaultOptionsToCS(gConfig,
'executor',
system,
executor,
getCSExtensions(),
specialOptions=specialOptions,
overwrite=overwrite,
addDefaultOptions=False)
else:
result = InstallTools.addDefaultOptionsToCS(gConfig,
'executor',
system,
executor,
getCSExtensions(),
Script.setUsageMessage(
'\n'.join(
[
__doc__.split('\n')[1],
'Usage:',
' %s [option|cfgfile] ... ComponentType System Component|System/Component' %
Script.scriptName,
'Arguments:',
' ComponentType: Name of the ComponentType (ie: agent)',
' System: Name of the DIRAC system (ie: WorkloadManagement)',
' component: Name of the DIRAC component (ie: JobCleaningAgent)']))
Script.parseCommandLine()
args = Script.getPositionalArgs()
componentType = args[0]
if len(args) == 2:
system, component = args[1].split('/')
else:
system = args[1]
component = args[2]
result = gComponentInstaller.addDefaultOptionsToCS(gConfig, componentType, system, component,
getCSExtensions(),
specialOptions={},
overwrite=False)
if not result['OK']:
print "ERROR:", result['Message']
else:
DIRACexit()
Script.parseCommandLine()
args = Script.getPositionalArgs()
if len(args) == 1:
args = args[0].split('/')
if len(args) != 2:
Script.showHelp()
DIRACexit(1)
cType = None
system = args[0]
component = args[1]
result = gComponentInstaller.getSoftwareComponents(getCSExtensions())
if not result['OK']:
gLogger.error(result['Message'])
DIRACexit(1)
else:
availableComponents = result['Value']
for compType in availableComponents:
if system in availableComponents[
compType] and component in availableComponents[compType][system]:
cType = compType[:-1].lower()
break
if not cType:
gLogger.error('Component %s/%s is not available for installation' %
(system, component))
' Agent: Name of the DIRAC agent (ie: JobCleaningAgent)'] ) )
Script.parseCommandLine()
args = Script.getPositionalArgs()
if len( args ) == 1:
args = args[0].split( '/' )
if len( args ) != 2:
Script.showHelp()
DIRACexit( -1 )
#
system = args[0]
agent = args[1]
if module:
result = InstallTools.addDefaultOptionsToCS( gConfig, 'agent', system, module,
getCSExtensions(),
overwrite = overwrite )
result = InstallTools.addDefaultOptionsToCS( gConfig, 'agent', system, agent,
getCSExtensions(),
specialOptions=specialOptions,
overwrite = overwrite,
addDefaultOptions = False )
else:
result = InstallTools.addDefaultOptionsToCS( gConfig, 'agent', system, agent,
getCSExtensions(),
specialOptions=specialOptions,
overwrite = overwrite )
if not result['OK']:
print "ERROR:", result['Message']
else:
result = InstallTools.installComponent( 'agent', system, agent, getCSExtensions(), module )
def do_install( self, args ):
"""
Install various DIRAC components
usage:
install mysql
install db <database>
install service <system> <service> [-m <ModuleName>] [-p <Option>=<Value>] [-p <Option>=<Value>] ...
install agent <system> <agent> [-m <ModuleName>] [-p <Option>=<Value>] [-p <Option>=<Value>] ...
install executor <system> <executor> [-m <ModuleName>] [-p <Option>=<Value>] [-p <Option>=<Value>] ...
"""
argss = args.split()
if not argss:
print self.do_install.__doc__
return
option = argss[0]
del argss[0]
if option == "mysql":
print "Installing MySQL database, this can take a while ..."
client = SystemAdministratorClient( self.host, self.port )
if InstallTools.mysqlPassword == 'LocalConfig':
InstallTools.mysqlPassword = ''
InstallTools.getMySQLPasswords()
result = client.installMySQL( InstallTools.mysqlRootPwd, InstallTools.mysqlPassword )
if not result['OK']:
self.__errMsg( result['Message'] )
else:
print "MySQL:", result['Value']
print "You might need to restart SystemAdministrator service to take new settings into account"
elif option == "db":
if not argss:
print self.do_install.__doc__
return
database = argss[0]
client = SystemAdministratorClient( self.host, self.port )
result = client.getAvailableDatabases()
if not result['OK']:
self.__errMsg( "Can not get database list: %s" % result['Message'] )
return
if not result['Value'].has_key( database ):
self.__errMsg( "Unknown database %s: " % database )
return
system = result['Value'][database]['System']
setup = gConfig.getValue( '/DIRAC/Setup', '' )
if not setup:
self.__errMsg( "Unknown current setup" )
return
instance = gConfig.getValue( '/DIRAC/Setups/%s/%s' % ( setup, system ), '' )
if not instance:
self.__errMsg( "No instance defined for system %s" % system )
self.__errMsg( "\tAdd new instance with 'add instance %s <instance_name>'" % system )
return
if not InstallTools.mysqlPassword:
InstallTools.mysqlPassword = '******'
InstallTools.getMySQLPasswords()
result = client.installDatabase( database, InstallTools.mysqlRootPwd )
if not result['OK']:
self.__errMsg( result['Message'] )
return
extension, system = result['Value']
# result = client.addDatabaseOptionsToCS( system, database )
InstallTools.mysqlHost = self.host
result = client.getInfo()
if not result['OK']:
self.__errMsg( result['Message'] )
hostSetup = result['Value']['Setup']
result = InstallTools.addDatabaseOptionsToCS( gConfig, system, database, hostSetup )
if not result['OK']:
self.__errMsg( result['Message'] )
return
print "Database %s from %s/%s installed successfully" % ( database, extension, system )
elif option in ["service","agent","executor"] :
if len( argss ) < 2:
print self.do_install.__doc__
return
system = argss[0]
del argss[0]
component = argss[0]
del argss[0]
specialOptions = {}
module = ''
for i in range(len(argss)):
if argss[i] == "-m":
specialOptions['Module'] = argss[i+1]
module = argss[i+1]
if argss[i] == "-p":
opt,value = argss[i+1].split('=')
specialOptions[opt] = value
if module == component:
module = ''
client = SystemAdministratorClient( self.host, self.port )
# First need to update the CS
# result = client.addDefaultOptionsToCS( option, system, component )
InstallTools.host = self.host
result = client.getInfo()
if not result['OK']:
self.__errMsg( result['Message'] )
return
hostSetup = result['Value']['Setup']
# Install Module section if not yet there
if module:
result = InstallTools.addDefaultOptionsToCS( gConfig, option, system, module,
getCSExtensions(), hostSetup )
# Add component section with specific parameters only
result = InstallTools.addDefaultOptionsToCS( gConfig, option, system, component,
getCSExtensions(), hostSetup, specialOptions,
addDefaultOptions = False )
else:
# Install component section
result = InstallTools.addDefaultOptionsToCS( gConfig, option, system, component,
getCSExtensions(), hostSetup, specialOptions )
if not result['OK']:
self.__errMsg( result['Message'] )
return
# Then we can install and start the component
result = client.setupComponent( option, system, component, module )
if not result['OK']:
self.__errMsg( result['Message'] )
return
compType = result['Value']['ComponentType']
runit = result['Value']['RunitStatus']
print "%s %s_%s is installed, runit status: %s" % ( compType, system, component, runit )
else:
print "Unknown option:", option
def main():
global overwrite
global specialOptions
global module
global specialOptions
from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller
gComponentInstaller.exitOnError = True
Script.registerSwitch("w", "overwrite",
"Overwrite the configuration in the global CS",
setOverwrite)
Script.registerSwitch("m:", "module=",
"Python module name for the component code",
setModule)
Script.registerSwitch("p:", "parameter=", "Special component option ",
setSpecialOption)
Script.parseCommandLine()
args = Script.getPositionalArgs()
if len(args) == 1:
args = args[0].split('/')
if len(args) != 2:
Script.showHelp(exitCode=1)
cType = None
system = args[0]
component = args[1]
compOrMod = module if module else component
result = gComponentInstaller.getSoftwareComponents(getCSExtensions())
if not result['OK']:
gLogger.error(result['Message'])
DIRACexit(1)
else:
availableComponents = result['Value']
for compType in availableComponents:
if system in availableComponents[
compType] and compOrMod in availableComponents[compType][
system]:
cType = compType[:-1].lower()
break
if not cType:
gLogger.error('Component %s/%s is not available for installation' %
(system, component))
DIRACexit(1)
if module:
result = gComponentInstaller.addDefaultOptionsToCS(gConfig,
cType,
system,
module,
getCSExtensions(),
overwrite=overwrite)
result = gComponentInstaller.addDefaultOptionsToCS(
gConfig,
cType,
system,
component,
getCSExtensions(),
specialOptions=specialOptions,
overwrite=overwrite,
addDefaultOptions=False)
else:
result = gComponentInstaller.addDefaultOptionsToCS(
gConfig,
cType,
system,
component,
getCSExtensions(),
specialOptions=specialOptions,
overwrite=overwrite)
if not result['OK']:
gLogger.error(result['Message'])
DIRACexit(1)
else:
result = gComponentInstaller.installComponent(cType, system, component,
getCSExtensions(),
module)
if not result['OK']:
gLogger.error(result['Message'])
DIRACexit(1)
else:
gLogger.notice(
'Successfully installed component %s in %s system, now setting it up'
% (component, system))
result = gComponentInstaller.setupComponent(
cType, system, component, getCSExtensions(), module)
if not result['OK']:
gLogger.error(result['Message'])
DIRACexit(1)
if component == 'ComponentMonitoring':
result = MonitoringUtilities.monitorInstallation(
'DB', system, 'InstalledComponentsDB')
if not result['OK']:
gLogger.error(result['Message'])
DIRACexit(1)
result = MonitoringUtilities.monitorInstallation(
cType, system, component, module)
if not result['OK']:
gLogger.error(result['Message'])
DIRACexit(1)
gLogger.notice('Successfully completed the installation of %s/%s' %
(system, component))
DIRACexit()
def _getPilotOptions(self, taskQueueDict, pilotsToSubmit):
# Need to limit the maximum number of pilots to submit at once
# For generic pilots this is limited by the number of use of the tokens and the
# maximum number of jobs in Filling mode, but for private Jobs we need an extra limitation:
pilotsToSubmit = max(
min(pilotsToSubmit, int(50 / self.maxJobsInFillMode)), 1)
pilotOptions = []
privateIfGenericTQ = self.privatePilotFraction > random.random()
privateTQ = ('PilotTypes' in taskQueueDict and 'private'
in [t.lower() for t in taskQueueDict['PilotTypes']])
forceGeneric = 'ForceGeneric' in taskQueueDict
submitPrivatePilot = (privateIfGenericTQ
or privateTQ) and not forceGeneric
if submitPrivatePilot:
self.log.verbose('Submitting private pilots for TaskQueue %s' %
taskQueueDict['TaskQueueID'])
ownerDN = taskQueueDict['OwnerDN']
ownerGroup = taskQueueDict['OwnerGroup']
# User Group requirement
pilotOptions.append('-G %s' % taskQueueDict['OwnerGroup'])
# check if group allows jobsharing
ownerGroupProperties = getPropertiesForGroup(ownerGroup)
if not 'JobSharing' in ownerGroupProperties:
# Add Owner requirement to pilot
pilotOptions.append("-O '%s'" % ownerDN)
if privateTQ:
pilotOptions.append(
'-o /Resources/Computing/CEDefaults/PilotType=private')
maxJobsInFillMode = self.maxJobsInFillMode
else:
#For generic jobs we'll submit mixture of generic and private pilots
self.log.verbose('Submitting generic pilots for TaskQueue %s' %
taskQueueDict['TaskQueueID'])
#ADRI: Find the generic group
result = findGenericPilotCredentials(
group=taskQueueDict['OwnerGroup'])
if not result['OK']:
self.log.error(ERROR_GENERIC_CREDENTIALS, result['Message'])
return S_ERROR(ERROR_GENERIC_CREDENTIALS)
ownerDN, ownerGroup = result['Value']
result = gProxyManager.requestToken(
ownerDN, ownerGroup, max(pilotsToSubmit,
self.maxJobsInFillMode))
if not result['OK']:
self.log.error(ERROR_TOKEN, result['Message'])
return S_ERROR(ERROR_TOKEN)
(token, numberOfUses) = result['Value']
pilotsToSubmit = min(numberOfUses, pilotsToSubmit)
pilotOptions.append('-o /Security/ProxyToken=%s' % token)
pilotsToSubmit = max(
1, (pilotsToSubmit - 1) / self.maxJobsInFillMode + 1)
maxJobsInFillMode = int(numberOfUses / pilotsToSubmit)
# Use Filling mode
pilotOptions.append('-M %s' % maxJobsInFillMode)
# Debug
pilotOptions.append('-d')
# Setup.
pilotOptions.append('-S %s' % taskQueueDict['Setup'])
# CS Servers
csServers = gConfig.getServersList()
if len(csServers) > 3:
# Remove the master
master = gConfigurationData.getMasterServer()
if master in csServers:
csServers.remove(master)
pilotOptions.append('-C %s' % ",".join(csServers))
# DIRAC Extensions to be used in pilots
# ubeda: I'm not entirely sure if we can use here the same opsHelper as in line
# line +352
pilotExtensionsList = Operations().getValue("Pilot/Extensions", [])
extensionsList = []
if pilotExtensionsList:
if pilotExtensionsList[0] != 'None':
extensionsList = pilotExtensionsList
else:
extensionsList = getCSExtensions()
if extensionsList:
pilotOptions.append('-e %s' % ",".join(extensionsList))
#Get DIRAC version and project, There might be global Setup defaults and per VO/Setup defaults (from configure)
opsHelper = Operations(group=taskQueueDict['OwnerGroup'],
setup=taskQueueDict['Setup'])
# Requested version of DIRAC (it can be a list, so we take the fist one)
version = opsHelper.getValue(cfgPath('Pilot', 'Version'),
[self.installVersion])[0]
pilotOptions.append('-r %s' % version)
# Requested Project to install
installProject = opsHelper.getValue(cfgPath('Pilot', 'Project'),
self.installProject)
if installProject:
pilotOptions.append('-l %s' % installProject)
installation = opsHelper.getValue(cfgPath('Pilot', 'Installation'),
self.installation)
if installation:
pilotOptions.append("-V %s" % installation)
# Requested CPU time
pilotOptions.append('-T %s' % taskQueueDict['CPUTime'])
if self.submitPoolOption not in self.extraPilotOptions:
pilotOptions.append(self.submitPoolOption)
if self.extraPilotOptions:
pilotOptions.extend(self.extraPilotOptions)
return S_OK((pilotOptions, pilotsToSubmit, ownerDN, ownerGroup,
submitPrivatePilot, privateTQ))
Script.parseCommandLine()
args = Script.getPositionalArgs()
if len(args) == 1:
args = args[0].split("/")
if len(args) != 2:
Script.showHelp()
DIRACexit(-1)
#
system = args[0]
executor = args[1]
if module:
result = InstallTools.addDefaultOptionsToCS(
gConfig, "executor", system, module, getCSExtensions(), overwrite=overwrite
)
result = InstallTools.addDefaultOptionsToCS(
gConfig,
"executor",
system,
executor,
getCSExtensions(),
specialOptions=specialOptions,
overwrite=overwrite,
addDefaultOptions=False,
)
else:
result = InstallTools.addDefaultOptionsToCS(
gConfig, "executor", system, executor, getCSExtensions(), specialOptions=specialOptions, overwrite=overwrite
)
def do_install(self, args):
"""
Install various DIRAC components
usage:
install mysql
install db <database>
install service <system> <service> [-m <ModuleName>] [-p <Option>=<Value>] [-p <Option>=<Value>] ...
install agent <system> <agent> [-m <ModuleName>] [-p <Option>=<Value>] [-p <Option>=<Value>] ...
install executor <system> <executor> [-m <ModuleName>] [-p <Option>=<Value>] [-p <Option>=<Value>] ...
"""
argss = args.split()
if not argss:
print self.do_install.__doc__
return
option = argss[0]
del argss[0]
if option == "mysql":
print "Installing MySQL database, this can take a while ..."
client = SystemAdministratorClient(self.host, self.port)
if InstallTools.mysqlPassword == 'LocalConfig':
InstallTools.mysqlPassword = ''
InstallTools.getMySQLPasswords()
result = client.installMySQL(InstallTools.mysqlRootPwd,
InstallTools.mysqlPassword)
if not result['OK']:
self.__errMsg(result['Message'])
else:
print "MySQL:", result['Value']
print "You might need to restart SystemAdministrator service to take new settings into account"
elif option == "db":
if not argss:
print self.do_install.__doc__
return
database = argss[0]
client = SystemAdministratorClient(self.host, self.port)
result = client.getAvailableDatabases()
if not result['OK']:
self.__errMsg("Can not get database list: %s" %
result['Message'])
return
if not result['Value'].has_key(database):
self.__errMsg("Unknown database %s: " % database)
return
system = result['Value'][database]['System']
setup = gConfig.getValue('/DIRAC/Setup', '')
if not setup:
self.__errMsg("Unknown current setup")
return
instance = gConfig.getValue(
'/DIRAC/Setups/%s/%s' % (setup, system), '')
if not instance:
self.__errMsg("No instance defined for system %s" % system)
self.__errMsg(
"\tAdd new instance with 'add instance %s <instance_name>'"
% system)
return
if not InstallTools.mysqlPassword:
InstallTools.mysqlPassword = '******'
InstallTools.getMySQLPasswords()
result = client.installDatabase(database,
InstallTools.mysqlRootPwd)
if not result['OK']:
self.__errMsg(result['Message'])
return
extension, system = result['Value']
# result = client.addDatabaseOptionsToCS( system, database )
InstallTools.mysqlHost = self.host
result = client.getInfo()
if not result['OK']:
self.__errMsg(result['Message'])
hostSetup = result['Value']['Setup']
result = InstallTools.addDatabaseOptionsToCS(
gConfig, system, database, hostSetup)
if not result['OK']:
self.__errMsg(result['Message'])
return
print "Database %s from %s/%s installed successfully" % (
database, extension, system)
elif option in ["service", "agent", "executor"]:
if len(argss) < 2:
print self.do_install.__doc__
return
system = argss[0]
del argss[0]
component = argss[0]
del argss[0]
specialOptions = {}
module = ''
for i in range(len(argss)):
if argss[i] == "-m":
specialOptions['Module'] = argss[i + 1]
module = argss[i + 1]
if argss[i] == "-p":
opt, value = argss[i + 1].split('=')
specialOptions[opt] = value
if module == component:
module = ''
client = SystemAdministratorClient(self.host, self.port)
# First need to update the CS
# result = client.addDefaultOptionsToCS( option, system, component )
InstallTools.host = self.host
result = client.getInfo()
if not result['OK']:
self.__errMsg(result['Message'])
return
hostSetup = result['Value']['Setup']
# Install Module section if not yet there
if module:
result = InstallTools.addDefaultOptionsToCS(
gConfig, option, system, module, getCSExtensions(),
hostSetup)
# Add component section with specific parameters only
result = InstallTools.addDefaultOptionsToCS(
gConfig,
option,
system,
component,
getCSExtensions(),
hostSetup,
specialOptions,
addDefaultOptions=False)
else:
# Install component section
result = InstallTools.addDefaultOptionsToCS(
gConfig, option, system, component, getCSExtensions(),
hostSetup, specialOptions)
if not result['OK']:
self.__errMsg(result['Message'])
return
# Then we can install and start the component
result = client.setupComponent(option, system, component, module)
if not result['OK']:
self.__errMsg(result['Message'])
return
compType = result['Value']['ComponentType']
runit = result['Value']['RunitStatus']
print "%s %s_%s is installed, runit status: %s" % (
compType, system, component, runit)
else:
print "Unknown option:", option
def initDIRAC(rootPath, enableDebug=False):
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
configDict = {'webConfig': {}}
configDict['webConfig']['dirac.webroot'] = rootPath
diracRootPath = os.path.realpath(os.path.dirname(
os.path.dirname(rootPath)))
configDict['webConfig']['dirac.root'] = diracRootPath
if diracRootPath not in sys.path:
sys.path.append(diracRootPath)
from DIRAC.FrameworkSystem.Client.Logger import gLogger
gLogger.registerBackends(['stderr'])
from DIRAC.Core.Base import Script
Script.registerSwitch("r", "reload", "Reload for pylons")
Script.localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate",
"yes")
Script.localCfg.addDefaultEntry("LogColor", True)
Script.initialize(script="Website",
ignoreErrors=True,
initializeMonitor=False)
gLogger._systemName = "Framework"
gLogger.initialize("Web", "/Website")
gLogger.setLevel("VERBOSE")
from DIRAC import gMonitor, gConfig, rootPath as droot
from DIRAC.Core.Utilities import CFG
from DIRAC.ConfigurationSystem.Client.Helpers import getCSExtensions
gMonitor.setComponentType(gMonitor.COMPONENT_WEB)
gMonitor.initialize()
gMonitor.registerActivity("pagesServed", "Pages served", "Framework",
"pages", gMonitor.OP_SUM)
gLogger.info("DIRAC Initialized")
configDict['portalVersion'] = portalVersion(rootPath)
gLogger.info("DIRAC portal version: %s" % configDict['portalVersion'])
extModules = ['%sDIRAC' % module for module in getCSExtensions()]
#Load web.cfg of modules
cfgFilePaths = [os.path.join(droot, "etc", "web.cfg")]
for extModule in extModules:
gLogger.info("Adding web.cfg for %s extension" % extModule)
extModulePath = os.path.join(diracRootPath, extModule)
webCFGPath = os.path.join(extModulePath, "Web", "web.cfg")
cfgFilePaths.append(webCFGPath)
for systemDir in os.listdir(extModulePath):
webCFGSystemPath = os.path.join(extModulePath, systemDir, "Web",
"web.cfg")
cfgFilePaths.append(webCFGSystemPath)
webCFG = CFG.CFG()
for webCFGPath in cfgFilePaths:
if not os.path.isfile(webCFGPath):
gLogger.warn("%s does not exist" % webCFGPath)
else:
gLogger.info("Loading %s" % webCFGPath)
modCFG = CFG.CFG().loadFromFile(webCFGPath)
if modCFG.getOption('Website/AbsoluteDefinition', False):
gLogger.info("CFG %s is absolute" % webCFGPath)
webCFG = modCFG
else:
webCFG = webCFG.mergeWith(modCFG)
gConfig.loadCFG(webCFG)
gLogger.showHeaders(True)
gLogger._gLogger__initialized = False
gLogger.initialize("Web", "/Website")
#Define the controllers, templates and public directories
for type in ('controllers', 'templates', 'public'):
configDict[type] = []
for extModule in extModules:
extModulePath = os.path.join(diracRootPath, extModule)
typePath = os.path.join(extModulePath, "Web", type)
if os.path.isdir(typePath):
gLogger.info("Adding %s path for module %s" %
(type, extModule))
configDict[type].append(typePath)
for systemDir in os.listdir(extModulePath):
systemTypePath = os.path.join(extModulePath, systemDir, "Web",
type)
if os.path.isdir(systemTypePath):
gLogger.info("Adding %s path for system %s in module %s" %
(type, systemDir, extModule))
configDict[type].append(systemTypePath)
#End of extensions
configDict[type].append(os.path.join(rootPath, type))
#Load debug.cfg?
if enableDebug:
debugCFGPath = os.path.join(rootPath, "debug.cfg")
if os.path.isfile(debugCFGPath):
gLogger.info("Loading debug cfg file at %s" % debugCFGPath)
gConfig.loadFile(debugCFGPath)
gLogger.info("Extension modules loaded")
return configDict
