def __getInstallFlags(self, infoDict=None):
"""Get the flags for installing inside the container."""
if not infoDict:
infoDict = {}
setup = infoDict.get("DefaultSetup")
if not setup:
setup = list(infoDict.get("Setups"))[0]
if not setup:
setup = gConfig.getValue("/DIRAC/Setup", "unknown")
setup = str(setup)
diracProject = "DIRAC"
project = str(infoDict.get("Project"))
if not project or project == "None":
diracProject = Operations.Operations(setup=setup).getValue(
"Pilot/Project", "") + diracProject
diracVersions = str(
infoDict["Setups"][setup].get("Version")).split(",")
if not diracVersions:
diracVersions = str(
infoDict["Setups"]["Defaults"].get("Version")).split(",")
if not diracVersions:
diracVersions = Operations.Operations(setup=setup).getValue(
"Pilot/Version", [])
version = diracVersions[0].strip()
return diracProject, version
def __getInstallFlags(self, infoDict=None):
""" Get the flags to pass to dirac-install.py inside the container.
Returns a string containing the command line flags.
"""
if not infoDict:
infoDict = {}
instOpts = []
setup = infoDict.get('DefaultSetup')
if not setup:
setup = list(infoDict.get('Setups'))[0]
if not setup:
setup = gConfig.getValue("/DIRAC/Setup", "unknown")
setup = str(setup)
installationName = str(infoDict.get('Installation'))
if not installationName or installationName == 'None':
installationName = Operations.Operations(setup=setup).getValue(
"Pilot/Installation", "")
if installationName:
instOpts.append('-V %s' % installationName)
diracVersions = str(
infoDict['Setups'][setup].get('Version')).split(',')
if not diracVersions:
diracVersions = str(
infoDict['Setups']['Defaults'].get('Version')).split(',')
if not diracVersions:
diracVersions = Operations.Operations(setup=setup).getValue(
"Pilot/Version", [])
instOpts.append("-r '%s'" % diracVersions[0].strip())
pilotExtensionsList = str(
infoDict['Setups'][setup].get('CommandExtensions')).split(',')
if not pilotExtensionsList:
pilotExtensionsList = str(infoDict['Setups']['Defaults'].get(
'CommandExtensions')).split(',')
if not pilotExtensionsList:
pilotExtensionsList = Operations.Operations(setup=setup).getValue(
"Pilot/Extensions", [])
extensionsList = []
if pilotExtensionsList:
if pilotExtensionsList[0] != 'None':
extensionsList = pilotExtensionsList
else:
extensionsList = CSGlobals.getCSExtensions()
if extensionsList:
instOpts.append(
"-e '%s'" %
','.join([ext for ext in extensionsList if 'Web' not in ext]))
if 'ContainerExtraOpts' in self.ceParameters:
instOpts.append(self.ceParameters['ContainerExtraOpts'])
return ' '.join(instOpts)
def findGenericCloudCredentials(vo=False, group=False):
if not group and not vo:
return S_ERROR(
"Need a group or a VO to determine the Generic cloud credentials")
if not vo:
vo = Registry.getVOForGroup(group)
if not vo:
return S_ERROR("Group %s does not have a VO associated" % group)
opsHelper = Operations.Operations(vo=vo)
cloudGroup = opsHelper.getValue("Cloud/GenericCloudGroup", "")
cloudDN = opsHelper.getValue("Cloud/GenericCloudDN", "")
if not cloudDN:
cloudUser = opsHelper.getValue("Cloud/GenericCloudUser", "")
if cloudUser:
result = Registry.getDNForUsername(cloudUser)
if result['OK']:
cloudDN = result['Value'][0]
if cloudDN and cloudGroup:
gLogger.verbose("Cloud credentials from CS: %s@%s" %
(cloudDN, cloudGroup))
result = gProxyManager.userHasProxy(cloudDN, cloudGroup, 86400)
if not result['OK']:
return S_ERROR("%s@%s has no proxy in ProxyManager")
return S_OK((cloudDN, cloudGroup))
return S_ERROR("Cloud credentials not found")
def __getInstallFlags(self):
""" Get the flags to pass to dirac-install.py inside the container.
Returns a string containing the command line flags.
"""
instOpts = []
setup = gConfig.getValue("/DIRAC/Setup", "unknown")
opsHelper = Operations.Operations(setup=setup)
installationName = opsHelper.getValue("Pilot/Installation", "")
if installationName:
instOpts.append('-V %s' % installationName)
diracVersions = opsHelper.getValue("Pilot/Version", [])
instOpts.append("-r '%s'" % diracVersions[0])
pyVer = "%u%u" % (sys.version_info.major, sys.version_info.minor)
instOpts.append("-i %s" % pyVer)
pilotExtensionsList = opsHelper.getValue("Pilot/Extensions", [])
extensionsList = []
if pilotExtensionsList:
if pilotExtensionsList[0] != 'None':
extensionsList = pilotExtensionsList
else:
extensionsList = CSGlobals.getCSExtensions()
if extensionsList:
instOpts.append(
"-e '%s'" %
','.join([ext for ext in extensionsList if 'Web' not in ext]))
lcgVer = opsHelper.getValue("Pilot/LCGBundleVersion", None)
if lcgVer:
instOpts.append("-g %s" % lcgVer)
if 'ContainerExtraOpts' in self.ceParameters:
instOpts.append(self.ceParameters['ContainerExtraOpts'])
return ' '.join(instOpts)
def findGenericCloudCredentials(vo=False, group=False):
"""Get the cloud credentials to use for a specific VO and/or group."""
if not group and not vo:
return S_ERROR(
"Need a group or a VO to determine the Generic cloud credentials")
if not vo:
vo = Registry.getVOForGroup(group)
if not vo:
return S_ERROR("Group %s does not have a VO associated" % group)
opsHelper = Operations.Operations(vo=vo)
cloudGroup = opsHelper.getValue("Cloud/GenericCloudGroup", "")
cloudDN = opsHelper.getValue("Cloud/GenericCloudDN", "")
if not cloudDN:
cloudUser = opsHelper.getValue("Cloud/GenericCloudUser", "")
if cloudUser:
result = Registry.getDNForUsername(cloudUser)
if result["OK"]:
cloudDN = result["Value"][0]
else:
return S_ERROR("Failed to find suitable CloudDN")
if cloudDN and cloudGroup:
gLogger.verbose("Cloud credentials from CS: %s@%s" %
(cloudDN, cloudGroup))
result = gProxyManager.userHasProxy(cloudDN, cloudGroup, 86400)
if not result["OK"]:
return result
return S_OK((cloudDN, cloudGroup))
return S_ERROR("Cloud credentials not found")
def findGenericPilotCredentials( vo = False, group = False ):
if not group and not vo:
return S_ERROR( "Need a group or a VO to determine the Generic pilot credentials" )
if not vo:
vo = Registry.getVOForGroup( group )
if not vo:
return S_ERROR( "Group %s does not have a VO associated" % group )
opsHelper = Operations.Operations( vo = vo )
pilotGroup = opsHelper.getValue( "Pilot/GenericPilotGroup", "" )
pilotDN = opsHelper.getValue( "Pilot/GenericPilotDN", "" )
if pilotDN and pilotGroup:
gLogger.verbose( "Pilot credentials have been defined in the CS. Using %s@%s" % ( pilotDN, pilotGroup ) )
result = gProxyManager.userHasProxy( pilotDN, pilotGroup, 86400 )
if not result[ 'OK' ]:
return S_ERROR( "%s@%s has no proxy uploaded to the ProxyManager" )
return S_OK( ( pilotDN, pilotGroup ) )
#Auto discover
gLogger.verbose( "Pilot credentials are not defined. Autodiscovering..." )
if pilotGroup:
pilotGroups = [ pilotGroup ]
else:
result = Registry.getGroupsWithProperty( Properties.GENERIC_PILOT )
if not result[ 'OK' ]:
return result
pilotGroups = []
groups = result[ 'Value' ]
if not groups:
return S_ERROR( "No group with %s property defined" % Properties.GENERIC_PILOT )
result = Registry.getGroupsForVO( vo )
if not result[ 'OK' ]:
return result
for voGroup in result[ 'Value' ]:
if voGroup in groups:
pilotGroups.append( voGroup )
if not pilotGroups:
return S_ERROR( "No group for VO %s is a generic pilot group" % vo )
for pilotGroup in pilotGroups:
DNs = Registry.getDNsInGroup( pilotGroup )
if not DNs:
continue
if pilotDN:
if pilotDN not in DNs:
continue
result = gProxyManager.userHasProxy( pilotDN, pilotGroup, 86400 )
if result[ 'OK' ] and result[ 'Value' ]:
gLogger.verbose( "Discovered pilot credentials are %s@%s" % ( pilotDN, pilotGroup ) )
return S_OK( ( pilotDN, pilotGroup ) )
else:
for DN in DNs:
result = gProxyManager.userHasProxy( DN, pilotGroup, 86400 )
if result[ 'OK' ] and result[ 'Value' ]:
gLogger.verbose( "Discovered pilot credentials are %s@%s" % ( DN, pilotGroup ) )
return S_OK( ( DN, pilotGroup ) )
if pilotDN:
return S_ERROR( "DN %s does not have group %s" % ( pilotDN, pilotGroups ) )
return S_ERROR( "No generic proxy in the Proxy Manager with groups %s" % pilotGroups )
def __getOpsHelper(self, setup=False, vo=False):
if not setup:
setup = self.srv_getClientSetup()
if not vo:
vo = Registry.getVOForGroup(self.getRemoteCredentials()['group'])
cKey = (vo, setup)
if cKey not in MatcherHandler.__opsCache:
MatcherHandler.__opsCache[cKey] = Operations.Operations(
vo=vo, setup=setup)
return MatcherHandler.__opsCache[cKey]
def initialize(self):
self.am_disableMonitoring()
# Init vars
self.runningPod = gConfig.getValue('/LocalSite/RunningPod')
self.log.info("Running pod name of the image is %s" % self.runningPod)
self.vmID = gConfig.getValue('/LocalSite/VMID')
self.__loadHistory = []
self.vmMinWorkingLoad = None
self.vmLoadAvgTimespan = None
self.vmJobWrappersLocation = None
self.haltPeriod = None
self.haltBeforeMargin = None
self.heartBeatPeriod = None
self.am_setOption("MaxCycles", 0)
self.am_setOption("PollingTime", 60)
# Discover net address
self.ipAddress = None
netData = Network.discoverInterfaces()
for iface in sorted(netData):
# Warning! On different clouds interface name may be different(eth, ens, ...)
if "eth" in iface or "ens" in iface:
self.ipAddress = netData[iface]['ip']
self.log.info("IP Address is %s" % self.ipAddress)
break
# getting the stop policy
self.op = Operations.Operations()
self.vmStopPolicy = self.op.getValue("Cloud/%s/VMStopPolicy",
'elastic')
self.log.info("vmStopPolicy is %s" % self.vmStopPolicy)
# Declare instance running
self.uniqueID = ''
result = virtualMachineDB.getUniqueIDByName(self.vmID)
if result['OK']:
self.uniqueID = result['Value']
result = self.__declareInstanceRunning()
if not result['OK']:
self.log.error("Could not declare instance running",
result['Message'])
self.__haltInstance()
return S_ERROR("Halting!")
self.__instanceInfo = result['Value']
# Get the cs config
result = self.__getCSConfig()
if not result['OK']:
return result
return S_OK()
def getPilotBootstrapParameters( vo = '', runningPod = '' ):
op = Operations.Operations( vo = vo )
result = op.getOptionsDict( 'Cloud' )
opParameters = {}
if result['OK']:
opParameters = result['Value']
opParameters['Project'] = op.getValue( 'Cloud/Project', 'DIRAC' )
opParameters['Version'] = op.getValue( 'Cloud/Version' )
opParameters['Setup'] = gConfig.getValue( '/DIRAC/Setup', 'unknown' )
result = op.getOptionsDict( 'Cloud/%s' % runningPod )
if result['OK']:
opParameters.update( result['Value'] )
return S_OK( opParameters )
def findGenericPilotCredentials(vo=False,
group=False,
pilotDN="",
pilotGroup=""):
"""Looks into the Operations/<>/Pilot section of CS to find the pilot credentials.
Then check if the user has a registered proxy in ProxyManager.
if pilotDN or pilotGroup are specified, use them
:param str vo: VO name
:param str group: group name
:param str pilotDN: pilot DN
:param str pilotGroup: pilot group
:return: S_OK(tuple)/S_ERROR()
"""
if not group and not vo:
return S_ERROR(
"Need a group or a VO to determine the Generic pilot credentials")
if not vo:
vo = Registry.getVOForGroup(group)
if not vo:
return S_ERROR("Group %s does not have a VO associated" % group)
opsHelper = Operations.Operations(vo=vo)
if not pilotGroup:
pilotGroup = opsHelper.getValue("Pilot/GenericPilotGroup", "")
if not pilotDN:
pilotDN = opsHelper.getValue("Pilot/GenericPilotDN", "")
if not pilotDN:
pilotUser = opsHelper.getValue("Pilot/GenericPilotUser", "")
if pilotUser:
result = Registry.getDNForUsername(pilotUser)
if result["OK"]:
pilotDN = result["Value"][0]
if pilotDN and pilotGroup:
gLogger.verbose("Pilot credentials: %s@%s" % (pilotDN, pilotGroup))
result = gProxyManager.userHasProxy(pilotDN, pilotGroup, 86400)
if not result["OK"]:
return S_ERROR("%s@%s has no proxy in ProxyManager")
return S_OK((pilotDN, pilotGroup))
if pilotDN:
return S_ERROR("DN %s does not have group %s" % (pilotDN, pilotGroup))
return S_ERROR("No generic proxy in the Proxy Manager with groups %s" %
pilotGroup)
def getPilotBootstrapParameters(vo='', runningPod=''):
op = Operations.Operations(vo=vo)
result = op.getOptionsDict('Cloud')
opParameters = {}
if result['OK']:
opParameters = result['Value']
opParameters['VO'] = vo
opParameters['ReleaseProject'] = op.getValue('Cloud/ReleaseProject',
'DIRAC')
opParameters['ReleaseVersion'] = op.getValue('Cloud/ReleaseVersion')
opParameters['Setup'] = gConfig.getValue('/DIRAC/Setup', 'unknown')
opParameters['SubmitPool'] = op.getValue('Cloud/SubmitPool')
opParameters['CloudPilotCert'] = op.getValue('Cloud/CloudPilotCert')
opParameters['CloudPilotKey'] = op.getValue('Cloud/CloudPilotKey')
result = op.getOptionsDict('Cloud/%s' % runningPod)
if result['OK']:
opParameters.update(result['Value'])
return S_OK(opParameters)
def getPilotBootstrapParameters(vo="", runningPod=""):
"""Get all of the settings required to bootstrap a cloud instance."""
op = Operations.Operations(vo=vo)
result = op.getOptionsDict("Cloud")
opParameters = {}
if result["OK"]:
opParameters = result["Value"]
opParameters["VO"] = vo
# FIXME: The majority of these settings can be removed once the old vm-pilot
# scripts have been removed.
opParameters["ReleaseProject"] = op.getValue("Cloud/ReleaseProject",
"DIRAC")
opParameters["ReleaseVersion"] = op.getValue("Cloud/ReleaseVersion",
op.getValue("Pilot/Version"))
opParameters["Setup"] = gConfig.getValue("/DIRAC/Setup", "unknown")
opParameters["SubmitPool"] = op.getValue("Cloud/SubmitPool")
opParameters["CloudPilotCert"] = op.getValue("Cloud/CloudPilotCert")
opParameters["CloudPilotKey"] = op.getValue("Cloud/CloudPilotKey")
opParameters["pilotFileServer"] = op.getValue("Pilot/pilotFileServer")
result = op.getOptionsDict("Cloud/%s" % runningPod)
if result["OK"]:
opParameters.update(result["Value"])
# Get standard pilot version now
if "Version" in opParameters:
gLogger.warn(
"Cloud bootstrap version now uses standard Pilot/Version setting. "
"Please remove all obsolete (Cloud/Version) setting(s).")
pilotVersions = op.getValue("Pilot/Version")
if isinstance(pilotVersions, six.string_types):
pilotVersions = [pilotVersions]
if not pilotVersions:
return S_ERROR("Failed to get pilot version.")
opParameters["Version"] = pilotVersions[0].strip()
return S_OK(opParameters)
def __getPilotOptions(self, queue, pilotsToSubmit):
""" Prepare pilot options
"""
queueDict = self.queueDict[queue]['ParametersDict']
pilotOptions = []
setup = gConfig.getValue("/DIRAC/Setup", "unknown")
if setup == 'unknown':
self.log.error('Setup is not defined in the configuration')
return [None, None]
pilotOptions.append('-S %s' % setup)
opsHelper = Operations.Operations(group=self.pilotGroup, setup=setup)
#Installation defined?
installationName = opsHelper.getValue("Pilot/Installation", "")
if installationName:
pilotOptions.append('-V %s' % installationName)
#Project defined?
projectName = opsHelper.getValue("Pilot/Project", "")
if projectName:
pilotOptions.append('-l %s' % projectName)
else:
self.log.info('DIRAC project will be installed by pilots')
#Request a release
diracVersion = opsHelper.getValue("Pilot/Version", [])
if not diracVersion:
self.log.error('Pilot/Version is not defined in the configuration')
return [None, None]
#diracVersion is a list of accepted releases. Just take the first one
pilotOptions.append('-r %s' % diracVersion[0])
ownerDN = self.pilotDN
ownerGroup = self.pilotGroup
# Request token for maximum pilot efficiency
result = gProxyManager.requestToken(
ownerDN, ownerGroup, pilotsToSubmit * self.maxJobsInFillMode)
if not result['OK']:
self.log.error('Invalid proxy token request', result['Message'])
return [None, None]
(token, numberOfUses) = result['Value']
pilotOptions.append('-o /Security/ProxyToken=%s' % token)
# Use Filling mode
pilotOptions.append('-M %s' %
min(numberOfUses, self.maxJobsInFillMode))
# Since each pilot will execute min( numberOfUses, self.maxJobsInFillMode )
# with numberOfUses tokens we can submit at most:
# numberOfUses / min( numberOfUses, self.maxJobsInFillMode )
# pilots
newPilotsToSubmit = numberOfUses / min(numberOfUses,
self.maxJobsInFillMode)
if newPilotsToSubmit != pilotsToSubmit:
self.log.info(
'Number of pilots to submit is changed to %d after getting the proxy token'
% newPilotsToSubmit)
pilotsToSubmit = newPilotsToSubmit
# Debug
if self.pilotLogLevel.lower() == 'debug':
pilotOptions.append('-d')
# CS Servers
csServers = gConfig.getValue("/DIRAC/Configuration/Servers", [])
pilotOptions.append('-C %s' % ",".join(csServers))
# DIRAC Extensions
extensionsList = CSGlobals.getCSExtensions()
if extensionsList:
pilotOptions.append('-e %s' % ",".join(extensionsList))
# Requested CPU time
pilotOptions.append('-T %s' % queueDict['CPUTime'])
# CEName
pilotOptions.append('-N %s' % self.queueDict[queue]['CEName'])
# SiteName
pilotOptions.append('-n %s' % queueDict['Site'])
if 'ClientPlatform' in queueDict:
pilotOptions.append("-p '%s'" % queueDict['ClientPlatform'])
if 'SharedArea' in queueDict:
pilotOptions.append("-o '/LocalSite/SharedArea=%s'" %
queueDict['SharedArea'])
if 'SI00' in queueDict:
factor = float(queueDict['SI00']) / 250.
pilotOptions.append("-o '/LocalSite/CPUScalingFactor=%s'" % factor)
pilotOptions.append("-o '/LocalSite/CPUNormalizationFactor=%s'" %
factor)
else:
if 'CPUScalingFactor' in queueDict:
pilotOptions.append("-o '/LocalSite/CPUScalingFactor=%s'" %
queueDict['CPUScalingFactor'])
if 'CPUNormalizationFactor' in queueDict:
pilotOptions.append(
"-o '/LocalSite/CPUNormalizationFactor=%s'" %
queueDict['CPUNormalizationFactor'])
# Hack
if self.defaultSubmitPools:
pilotOptions.append(
'-o /Resources/Computing/CEDefaults/SubmitPool=%s' %
self.defaultSubmitPools)
if self.group:
pilotOptions.append('-G %s' % self.group)
self.log.verbose("pilotOptions: ", ' '.join(pilotOptions))
return [pilotOptions, pilotsToSubmit]
