def getQueues( self, resourceDict ):
""" Get the list of relevant CEs and their descriptions
"""
self.queueDict = {}
ceFactory = ComputingElementFactory()
for site in resourceDict:
for ce in resourceDict[site]:
ceDict = resourceDict[site][ce]
ceTags = ceDict.get( 'Tag' )
if isinstance( ceTags, basestring ):
ceTags = fromChar( ceTags )
qDict = ceDict.pop( 'Queues' )
for queue in qDict:
queueName = '%s_%s' % ( ce, queue )
self.queueDict[queueName] = {}
self.queueDict[queueName]['ParametersDict'] = qDict[queue]
self.queueDict[queueName]['ParametersDict']['Queue'] = queue
self.queueDict[queueName]['ParametersDict']['Site'] = site
self.queueDict[queueName]['ParametersDict']['GridEnv'] = self.gridEnv
self.queueDict[queueName]['ParametersDict']['Setup'] = gConfig.getValue( '/DIRAC/Setup', 'unknown' )
# Evaluate the CPU limit of the queue according to the Glue convention
# To Do: should be a utility
if "maxCPUTime" in self.queueDict[queueName]['ParametersDict'] and \
"SI00" in self.queueDict[queueName]['ParametersDict']:
maxCPUTime = float( self.queueDict[queueName]['ParametersDict']['maxCPUTime'] )
# For some sites there are crazy values in the CS
maxCPUTime = max( maxCPUTime, 0 )
maxCPUTime = min( maxCPUTime, 86400 * 12.5 )
si00 = float( self.queueDict[queueName]['ParametersDict']['SI00'] )
queueCPUTime = 60. / 250. * maxCPUTime * si00
self.queueDict[queueName]['ParametersDict']['CPUTime'] = int( queueCPUTime )
queueTags = self.queueDict[queueName]['ParametersDict'].get( 'Tag' )
if queueTags and isinstance( queueTags, basestring ):
queueTags = fromChar( queueTags )
self.queueDict[queueName]['ParametersDict']['Tag'] = queueTags
if ceTags:
if queueTags:
allTags = list( set( ceTags + queueTags ) )
self.queueDict[queueName]['ParametersDict']['Tag'] = allTags
else:
self.queueDict[queueName]['ParametersDict']['Tag'] = ceTags
maxMemory = self.queueDict[queueName]['ParametersDict'].get( 'MaxRAM', None )
if maxMemory:
# MaxRAM value is supposed to be in MB
maxMemoryList = range( 1, int( maxMemory )/1000 + 1 )
memoryTags = [ '%dGB' % mem for mem in maxMemoryList ]
if memoryTags:
self.queueDict[queueName]['ParametersDict'].setdefault( 'Tag', [] )
self.queueDict[queueName]['ParametersDict']['Tag'] += memoryTags
qwDir = os.path.join( self.workingDirectory, queue )
if not os.path.exists( qwDir ):
os.makedirs( qwDir )
self.queueDict[queueName]['ParametersDict']['WorkingDirectory'] = qwDir
platform = ''
if "Platform" in self.queueDict[queueName]['ParametersDict']:
platform = self.queueDict[queueName]['ParametersDict']['Platform']
elif "Platform" in ceDict:
platform = ceDict['Platform']
elif "OS" in ceDict:
architecture = ceDict.get( 'architecture', 'x86_64' )
OS = ceDict['OS']
platform = '_'.join( [architecture, OS] )
if platform and not platform in self.platforms:
self.platforms.append( platform )
if not "Platform" in self.queueDict[queueName]['ParametersDict'] and platform:
result = Resources.getDIRACPlatform( platform )
if result['OK']:
self.queueDict[queueName]['ParametersDict']['Platform'] = result['Value'][0]
ceQueueDict = dict( ceDict )
ceQueueDict.update( self.queueDict[queueName]['ParametersDict'] )
# Generate the CE object for the queue or pick the already existing one
# if the queue definition did not change
queueHash = self.__generateQueueHash( ceQueueDict )
if queueName in self.queueCECache and self.queueCECache[queueName]['Hash'] == queueHash:
queueCE = self.queueCECache[queueName]['CE']
else:
result = ceFactory.getCE( ceName = ce,
ceType = ceDict['CEType'],
ceParametersDict = ceQueueDict )
if not result['OK']:
return result
self.queueCECache.setdefault( queueName, {} )
self.queueCECache[queueName]['Hash'] = queueHash
self.queueCECache[queueName]['CE'] = result['Value']
queueCE = self.queueCECache[queueName]['CE']
self.queueDict[queueName]['CE'] = queueCE
self.queueDict[queueName]['CEName'] = ce
self.queueDict[queueName]['CEType'] = ceDict['CEType']
self.queueDict[queueName]['Site'] = site
self.queueDict[queueName]['QueueName'] = queue
self.queueDict[queueName]['Platform'] = platform
result = self.queueDict[queueName]['CE'].isValid()
if not result['OK']:
self.log.fatal( result['Message'] )
return result
if 'BundleProxy' in self.queueDict[queueName]['ParametersDict']:
if self.queueDict[queueName]['ParametersDict']['BundleProxy'].lower() in ['true','yes','1']:
self.queueDict[queueName]['BundleProxy'] = True
elif 'BundleProxy' in ceDict:
if ceDict['BundleProxy'].lower() in ['true','yes','1']:
self.queueDict[queueName]['BundleProxy'] = True
if site not in self.sites:
self.sites.append( site )
return S_OK()
def getQueues( self, resourceDict ):
""" Get the list of relevant CEs and their descriptions
"""
self.queueDict = {}
ceFactory = ComputingElementFactory()
for site in resourceDict:
result = self._resources.getSiteFullName( site )
if not result['OK']:
continue
siteFullName = result['Value']
for ce in resourceDict[site]:
ceDict = resourceDict[site][ce]
qDict = ceDict.pop( 'Queues' )
for queue in qDict:
queueName = '%s_%s' % ( ce, queue )
self.queueDict[queueName] = {}
self.queueDict[queueName]['ParametersDict'] = qDict[queue]
self.queueDict[queueName]['ParametersDict']['Queue'] = queue
self.queueDict[queueName]['ParametersDict']['Site'] = siteFullName
self.queueDict[queueName]['ParametersDict']['GridEnv'] = self.gridEnv
self.queueDict[queueName]['ParametersDict']['Setup'] = gConfig.getValue( '/DIRAC/Setup', 'unknown' )
# Evaluate the CPU limit of the queue according to the Glue convention
# To Do: should be a utility
if "maxCPUTime" in self.queueDict[queueName]['ParametersDict'] and \
"SI00" in self.queueDict[queueName]['ParametersDict']:
maxCPUTime = float( self.queueDict[queueName]['ParametersDict']['maxCPUTime'] )
# For some sites there are crazy values in the CS
maxCPUTime = max( maxCPUTime, 0 )
maxCPUTime = min( maxCPUTime, 86400 * 12.5 )
si00 = float( self.queueDict[queueName]['ParametersDict']['SI00'] )
queueCPUTime = 60. / 250. * maxCPUTime * si00
self.queueDict[queueName]['ParametersDict']['CPUTime'] = int( queueCPUTime )
qwDir = os.path.join( self.workingDirectory, queue )
if not os.path.exists( qwDir ):
os.makedirs( qwDir )
self.queueDict[queueName]['ParametersDict']['WorkingDirectory'] = qwDir
platform = ''
if "Platform" in self.queueDict[queueName]['ParametersDict']:
platform = self.queueDict[queueName]['ParametersDict']['Platform']
elif "Platform" in ceDict:
platform = ceDict['Platform']
elif "OS" in ceDict:
architecture = ceDict.get( 'architecture', 'x86_64' )
OS = ceDict['OS']
platform = '_'.join( [architecture, OS] )
if platform and not platform in self.platforms:
self.platforms.append( platform )
if not "Platform" in self.queueDict[queueName]['ParametersDict'] and platform:
result = Resources.getDIRACPlatform( platform )
if result['OK']:
self.queueDict[queueName]['ParametersDict']['Platform'] = result['Value']
ceQueueDict = dict( ceDict )
ceQueueDict.update( self.queueDict[queueName]['ParametersDict'] )
result = ceFactory.getCE( ceName = ce,
ceType = ceDict['CEType'],
ceParametersDict = ceQueueDict )
if not result['OK']:
return result
self.queueDict[queueName]['CE'] = result['Value']
self.queueDict[queueName]['CEName'] = ce
self.queueDict[queueName]['CEType'] = ceDict['CEType']
self.queueDict[queueName]['Site'] = siteFullName
self.queueDict[queueName]['QueueName'] = queue
self.queueDict[queueName]['Platform'] = platform
result = self.queueDict[queueName]['CE'].isValid()
if not result['OK']:
self.log.fatal( result['Message'] )
return result
if 'BundleProxy' in self.queueDict[queueName]['ParametersDict']:
self.queueDict[queueName]['BundleProxy'] = True
elif 'BundleProxy' in ceDict:
self.queueDict[queueName]['BundleProxy'] = True
if siteFullName not in self.sites:
self.sites.append( siteFullName )
return S_OK()
def getQueues(self, resourceDict):
""" Get the list of relevant CEs and their descriptions
"""
self.queueDict = {}
ceFactory = ComputingElementFactory()
for site in resourceDict:
for ce in resourceDict[site]:
ceDict = resourceDict[site][ce]
qDict = ceDict.pop('Queues')
for queue in qDict:
queueName = '%s_%s' % (ce, queue)
self.queueDict[queueName] = {}
self.queueDict[queueName]['ParametersDict'] = qDict[queue]
self.queueDict[queueName]['ParametersDict'][
'Queue'] = queue
self.queueDict[queueName]['ParametersDict']['Site'] = site
self.queueDict[queueName]['ParametersDict'][
'GridEnv'] = self.gridEnv
self.queueDict[queueName]['ParametersDict'][
'Setup'] = gConfig.getValue('/DIRAC/Setup', 'unknown')
# Evaluate the CPU limit of the queue according to the Glue convention
# To Do: should be a utility
if "maxCPUTime" in self.queueDict[queueName]['ParametersDict'] and \
"SI00" in self.queueDict[queueName]['ParametersDict']:
maxCPUTime = float(self.queueDict[queueName]
['ParametersDict']['maxCPUTime'])
# For some sites there are crazy values in the CS
maxCPUTime = max(maxCPUTime, 0)
maxCPUTime = min(maxCPUTime, 86400 * 12.5)
si00 = float(self.queueDict[queueName]
['ParametersDict']['SI00'])
queueCPUTime = 60. / 250. * maxCPUTime * si00
self.queueDict[queueName]['ParametersDict'][
'CPUTime'] = int(queueCPUTime)
qwDir = os.path.join(self.workingDirectory, queue)
if not os.path.exists(qwDir):
os.makedirs(qwDir)
self.queueDict[queueName]['ParametersDict'][
'WorkingDirectory'] = qwDir
platform = ''
if "Platform" in self.queueDict[queueName][
'ParametersDict']:
platform = self.queueDict[queueName]['ParametersDict'][
'Platform']
elif "Platform" in ceDict:
platform = ceDict['Platform']
elif "OS" in ceDict:
architecture = ceDict.get('architecture', 'x86_64')
OS = ceDict['OS']
platform = '_'.join([architecture, OS])
if platform and not platform in self.platforms:
self.platforms.append(platform)
if not "Platform" in self.queueDict[queueName][
'ParametersDict'] and platform:
result = Resources.getDIRACPlatform(platform)
if result['OK']:
self.queueDict[queueName]['ParametersDict'][
'Platform'] = result['Value']
ceQueueDict = dict(ceDict)
ceQueueDict.update(
self.queueDict[queueName]['ParametersDict'])
result = ceFactory.getCE(ceName=ce,
ceType=ceDict['CEType'],
ceParametersDict=ceQueueDict)
if not result['OK']:
return result
self.queueDict[queueName]['CE'] = result['Value']
self.queueDict[queueName]['CEName'] = ce
self.queueDict[queueName]['CEType'] = ceDict['CEType']
self.queueDict[queueName]['Site'] = site
self.queueDict[queueName]['QueueName'] = queue
result = self.queueDict[queueName]['CE'].isValid()
if not result['OK']:
self.log.fatal(result['Message'])
return result
if 'BundleProxy' in self.queueDict[queueName][
'ParametersDict']:
self.queueDict[queueName]['BundleProxy'] = True
elif 'BundleProxy' in ceDict:
self.queueDict[queueName]['BundleProxy'] = True
if site not in self.sites:
self.sites.append(site)
return S_OK()
def getEndpoints(self, resourceDict):
""" Get the list of relevant CEs and their descriptions
"""
self.vmTypeDict = {}
ceFactory = EndpointFactory()
result = getPilotBootstrapParameters(vo=self.vo,
runningPod=self.runningPod)
if not result['OK']:
return result
opParameters = result['Value']
for site in resourceDict:
for ce in resourceDict[site]:
ceDict = resourceDict[site][ce]
ceTags = ceDict.get('Tag', [])
if isinstance(ceTags, basestring):
ceTags = fromChar(ceTags)
ceMaxRAM = ceDict.get('MaxRAM', None)
qDict = ceDict.pop('VMTypes')
for vmType in qDict:
vmTypeName = '%s_%s' % (ce, vmType)
self.vmTypeDict[vmTypeName] = {}
self.vmTypeDict[vmTypeName]['ParametersDict'] = qDict[
vmType]
self.vmTypeDict[vmTypeName]['ParametersDict'][
'VMType'] = vmType
self.vmTypeDict[vmTypeName]['ParametersDict'][
'Site'] = site
self.vmTypeDict[vmTypeName]['ParametersDict'][
'Setup'] = gConfig.getValue('/DIRAC/Setup', 'unknown')
self.vmTypeDict[vmTypeName]['ParametersDict'][
'CPUTime'] = 99999999
vmTypeTags = self.vmTypeDict[vmTypeName][
'ParametersDict'].get('Tag')
if vmTypeTags and isinstance(vmTypeTags, basestring):
vmTypeTags = fromChar(vmTypeTags)
self.vmTypeDict[vmTypeName]['ParametersDict'][
'Tag'] = vmTypeTags
if ceTags:
if vmTypeTags:
allTags = list(set(ceTags + vmTypeTags))
self.vmTypeDict[vmTypeName]['ParametersDict'][
'Tag'] = allTags
else:
self.vmTypeDict[vmTypeName]['ParametersDict'][
'Tag'] = ceTags
maxRAM = self.vmTypeDict[vmTypeName]['ParametersDict'].get(
'MaxRAM')
maxRAM = ceMaxRAM if not maxRAM else maxRAM
if maxRAM:
self.vmTypeDict[vmTypeName]['ParametersDict'][
'MaxRAM'] = maxRAM
ceWholeNode = ceDict.get('WholeNode', 'true')
wholeNode = self.vmTypeDict[vmTypeName][
'ParametersDict'].get('WholeNode', ceWholeNode)
if wholeNode.lower() in ('yes', 'true'):
self.vmTypeDict[vmTypeName][
'ParametersDict'].setdefault('Tag', [])
self.vmTypeDict[vmTypeName]['ParametersDict'][
'Tag'].append('WholeNode')
platform = ''
if "Platform" in self.vmTypeDict[vmTypeName][
'ParametersDict']:
platform = self.vmTypeDict[vmTypeName][
'ParametersDict']['Platform']
elif "Platform" in ceDict:
platform = ceDict['Platform']
if platform and platform not in self.platforms:
self.platforms.append(platform)
if "Platform" not in self.vmTypeDict[vmTypeName][
'ParametersDict'] and platform:
result = Resources.getDIRACPlatform(platform)
if result['OK']:
self.vmTypeDict[vmTypeName]['ParametersDict'][
'Platform'] = result['Value'][0]
ceVMTypeDict = dict(ceDict)
ceVMTypeDict['CEName'] = ce
ceVMTypeDict['VO'] = self.vo
ceVMTypeDict['VMType'] = vmType
ceVMTypeDict['RunningPod'] = self.runningPod
ceVMTypeDict['CSServers'] = gConfig.getValue(
"/DIRAC/Configuration/Servers", [])
ceVMTypeDict.update(
self.vmTypeDict[vmTypeName]['ParametersDict'])
# Allow a resource-specifc CAPath to be set (as some clouds have their own CAs)
# Otherwise fall back to the system-wide default(s)
if 'CAPath' not in ceVMTypeDict:
ceVMTypeDict['CAPath'] = gConfig.getValue(
'/DIRAC/Security/CAPath',
"/opt/dirac/etc/grid-security/certificates/cas.pem"
)
# Generate the CE object for the vmType or pick the already existing one
# if the vmType definition did not change
vmTypeHash = self.__generateVMTypeHash(ceVMTypeDict)
if vmTypeName in self.vmTypeCECache and self.vmTypeCECache[
vmTypeName]['Hash'] == vmTypeHash:
vmTypeCE = self.vmTypeCECache[vmTypeName]['CE']
else:
result = ceFactory.getCEObject(parameters=ceVMTypeDict)
if not result['OK']:
return result
self.vmTypeCECache.setdefault(vmTypeName, {})
self.vmTypeCECache[vmTypeName]['Hash'] = vmTypeHash
self.vmTypeCECache[vmTypeName]['CE'] = result['Value']
vmTypeCE = self.vmTypeCECache[vmTypeName]['CE']
vmTypeCE.setBootstrapParameters(opParameters)
self.vmTypeDict[vmTypeName]['CE'] = vmTypeCE
self.vmTypeDict[vmTypeName]['CEName'] = ce
self.vmTypeDict[vmTypeName]['CEType'] = ceDict['CEType']
self.vmTypeDict[vmTypeName]['Site'] = site
self.vmTypeDict[vmTypeName]['VMType'] = vmType
self.vmTypeDict[vmTypeName]['Platform'] = platform
self.vmTypeDict[vmTypeName]['MaxInstances'] = ceDict[
'MaxInstances']
if not self.vmTypeDict[vmTypeName]['CE'].isValid():
self.log.error(
'Failed to instantiate CloudEndpoint for %s' %
vmTypeName)
continue
if site not in self.sites:
self.sites.append(site)
return S_OK()
def getEndpoints(self, resourceDict):
"""Get the list of relevant CEs and their descriptions"""
self.vmTypeDict = {}
ceFactory = EndpointFactory()
result = getPilotBootstrapParameters(vo=self.vo, runningPod=self.runningPod)
if not result["OK"]:
return result
opParameters = result["Value"]
for site in resourceDict:
for ce in resourceDict[site]:
ceDict = resourceDict[site][ce]
ceTags = ceDict.get("Tag", [])
if isinstance(ceTags, six.string_types):
ceTags = fromChar(ceTags)
ceMaxRAM = ceDict.get("MaxRAM", None)
qDict = ceDict.pop("VMTypes")
for vmType in qDict:
vmTypeName = "%s_%s" % (ce, vmType)
self.vmTypeDict[vmTypeName] = {}
self.vmTypeDict[vmTypeName]["ParametersDict"] = qDict[vmType]
self.vmTypeDict[vmTypeName]["ParametersDict"]["VMType"] = vmType
self.vmTypeDict[vmTypeName]["ParametersDict"]["Site"] = site
self.vmTypeDict[vmTypeName]["ParametersDict"]["Setup"] = gConfig.getValue("/DIRAC/Setup", "unknown")
self.vmTypeDict[vmTypeName]["ParametersDict"]["CPUTime"] = 99999999
vmTypeTags = self.vmTypeDict[vmTypeName]["ParametersDict"].get("Tag")
if vmTypeTags and isinstance(vmTypeTags, six.string_types):
vmTypeTags = fromChar(vmTypeTags)
self.vmTypeDict[vmTypeName]["ParametersDict"]["Tag"] = vmTypeTags
if ceTags:
if vmTypeTags:
allTags = list(set(ceTags + vmTypeTags))
self.vmTypeDict[vmTypeName]["ParametersDict"]["Tag"] = allTags
else:
self.vmTypeDict[vmTypeName]["ParametersDict"]["Tag"] = ceTags
maxRAM = self.vmTypeDict[vmTypeName]["ParametersDict"].get("MaxRAM")
maxRAM = ceMaxRAM if not maxRAM else maxRAM
if maxRAM:
self.vmTypeDict[vmTypeName]["ParametersDict"]["MaxRAM"] = maxRAM
ceWholeNode = ceDict.get("WholeNode", "true")
wholeNode = self.vmTypeDict[vmTypeName]["ParametersDict"].get("WholeNode", ceWholeNode)
if wholeNode.lower() in ("yes", "true"):
self.vmTypeDict[vmTypeName]["ParametersDict"].setdefault("Tag", [])
self.vmTypeDict[vmTypeName]["ParametersDict"]["Tag"].append("WholeNode")
platform = ""
if "Platform" in self.vmTypeDict[vmTypeName]["ParametersDict"]:
platform = self.vmTypeDict[vmTypeName]["ParametersDict"]["Platform"]
elif "Platform" in ceDict:
platform = ceDict["Platform"]
if platform and platform not in self.platforms:
self.platforms.append(platform)
if "Platform" not in self.vmTypeDict[vmTypeName]["ParametersDict"] and platform:
result = Resources.getDIRACPlatform(platform)
if result["OK"]:
self.vmTypeDict[vmTypeName]["ParametersDict"]["Platform"] = result["Value"][0]
ceVMTypeDict = dict(ceDict)
ceVMTypeDict["CEName"] = ce
ceVMTypeDict["VO"] = self.vo
ceVMTypeDict["VMType"] = vmType
ceVMTypeDict["RunningPod"] = self.runningPod
ceVMTypeDict["CSServers"] = gConfig.getValue("/DIRAC/Configuration/Servers", [])
ceVMTypeDict.update(self.vmTypeDict[vmTypeName]["ParametersDict"])
# Allow a resource-specifc CAPath to be set (as some clouds have their own CAs)
# Otherwise fall back to the system-wide default(s)
if "CAPath" not in ceVMTypeDict:
ceVMTypeDict["CAPath"] = gConfig.getValue(
"/DIRAC/Security/CAPath", "/opt/dirac/etc/grid-security/certificates/cas.pem"
)
# Generate the CE object for the vmType or pick the already existing one
# if the vmType definition did not change
vmTypeHash = self.__generateVMTypeHash(ceVMTypeDict)
if vmTypeName in self.vmTypeCECache and self.vmTypeCECache[vmTypeName]["Hash"] == vmTypeHash:
vmTypeCE = self.vmTypeCECache[vmTypeName]["CE"]
else:
result = ceFactory.getCEObject(parameters=ceVMTypeDict)
if not result["OK"]:
return result
self.vmTypeCECache.setdefault(vmTypeName, {})
self.vmTypeCECache[vmTypeName]["Hash"] = vmTypeHash
self.vmTypeCECache[vmTypeName]["CE"] = result["Value"]
vmTypeCE = self.vmTypeCECache[vmTypeName]["CE"]
vmTypeCE.setBootstrapParameters(opParameters)
self.vmTypeDict[vmTypeName]["CE"] = vmTypeCE
self.vmTypeDict[vmTypeName]["CEName"] = ce
self.vmTypeDict[vmTypeName]["CEType"] = ceDict["CEType"]
self.vmTypeDict[vmTypeName]["Site"] = site
self.vmTypeDict[vmTypeName]["VMType"] = vmType
self.vmTypeDict[vmTypeName]["Platform"] = platform
self.vmTypeDict[vmTypeName]["MaxInstances"] = ceDict["MaxInstances"]
if not self.vmTypeDict[vmTypeName]["CE"].isValid():
self.log.error("Failed to instantiate CloudEndpoint for %s" % vmTypeName)
continue
if site not in self.sites:
self.sites.append(site)
return S_OK()
def getImages(self, resourceDict):
""" Get the list of relevant CEs and their descriptions
"""
self.imageDict = {}
ceFactory = EndpointFactory()
result = getPilotBootstrapParameters(vo=self.vo,
runningPod=self.runningPod)
if not result['OK']:
return result
opParameters = result['Value']
for site in resourceDict:
for ce in resourceDict[site]:
ceDict = resourceDict[site][ce]
ceTags = ceDict.get('Tag', [])
if isinstance(ceTags, basestring):
ceTags = fromChar(ceTags)
ceMaxRAM = ceDict.get('MaxRAM', None)
qDict = ceDict.pop('Images')
for image in qDict:
imageName = '%s_%s' % (ce, image)
self.imageDict[imageName] = {}
self.imageDict[imageName]['ParametersDict'] = qDict[image]
self.imageDict[imageName]['ParametersDict'][
'Image'] = image
self.imageDict[imageName]['ParametersDict']['Site'] = site
self.imageDict[imageName]['ParametersDict'][
'Setup'] = gConfig.getValue('/DIRAC/Setup', 'unknown')
self.imageDict[imageName]['ParametersDict'][
'CPUTime'] = 99999999
imageTags = self.imageDict[imageName][
'ParametersDict'].get('Tag')
if imageTags and isinstance(imageTags, basestring):
imageTags = fromChar(imageTags)
self.imageDict[imageName]['ParametersDict'][
'Tag'] = imageTags
if ceTags:
if imageTags:
allTags = list(set(ceTags + imageTags))
self.imageDict[imageName]['ParametersDict'][
'Tag'] = allTags
else:
self.imageDict[imageName]['ParametersDict'][
'Tag'] = ceTags
maxRAM = self.imageDict[imageName]['ParametersDict'].get(
'MaxRAM')
maxRAM = ceMaxRAM if not maxRAM else maxRAM
if maxRAM:
self.imageDict[imageName]['ParametersDict'][
'MaxRAM'] = maxRAM
platform = ''
if "Platform" in self.imageDict[imageName][
'ParametersDict']:
platform = self.imageDict[imageName]['ParametersDict'][
'Platform']
elif "Platform" in ceDict:
platform = ceDict['Platform']
if platform and not platform in self.platforms:
self.platforms.append(platform)
if not "Platform" in self.imageDict[imageName][
'ParametersDict'] and platform:
result = Resources.getDIRACPlatform(platform)
if result['OK']:
self.imageDict[imageName]['ParametersDict'][
'Platform'] = result['Value'][0]
ceImageDict = dict(ceDict)
ceImageDict['CEName'] = ce
ceImageDict['VO'] = self.vo
ceImageDict['Image'] = image
ceImageDict['RunningPod'] = self.runningPod
ceImageDict['CSServers'] = gConfig.getValue(
"/DIRAC/Configuration/Servers", [])
ceImageDict.update(
self.imageDict[imageName]['ParametersDict'])
ceImageDict.update(opParameters)
# Generate the CE object for the image or pick the already existing one
# if the image definition did not change
imageHash = self.__generateImageHash(ceImageDict)
if imageName in self.imageCECache and self.imageCECache[
imageName]['Hash'] == imageHash:
imageCE = self.imageCECache[imageName]['CE']
else:
result = ceFactory.getCEObject(parameters=ceImageDict)
if not result['OK']:
return result
self.imageCECache.setdefault(imageName, {})
self.imageCECache[imageName]['Hash'] = imageHash
self.imageCECache[imageName]['CE'] = result['Value']
imageCE = self.imageCECache[imageName]['CE']
self.imageDict[imageName]['CE'] = imageCE
self.imageDict[imageName]['CEName'] = ce
self.imageDict[imageName]['CEType'] = ceDict['CEType']
self.imageDict[imageName]['Site'] = site
self.imageDict[imageName]['ImageName'] = image
self.imageDict[imageName]['Platform'] = platform
self.imageDict[imageName]['MaxInstances'] = ceDict[
'MaxInstances']
if not self.imageDict[imageName]['CE'].isValid():
self.log.error(
'Failed to instantiate CloudEndpoint for %s' %
imageName)
continue
if site not in self.sites:
self.sites.append(site)
return S_OK()
def getQueues( self, resourceDict ):
""" Get the list of relevant CEs and their descriptions
"""
self.queueDict = {}
ceFactory = ComputingElementFactory()
for site in resourceDict:
for ce in resourceDict[site]:
ceDict = resourceDict[site][ce]
ceTags = ceDict.get( 'Tag', [] )
pilotRunDirectory = ceDict.get( 'PilotRunDirectory', '' )
if isinstance( ceTags, basestring ):
ceTags = fromChar( ceTags )
ceMaxRAM = ceDict.get( 'MaxRAM', None )
qDict = ceDict.pop( 'Queues' )
for queue in qDict:
queueName = '%s_%s' % ( ce, queue )
self.queueDict[queueName] = {}
self.queueDict[queueName]['ParametersDict'] = qDict[queue]
self.queueDict[queueName]['ParametersDict']['Queue'] = queue
self.queueDict[queueName]['ParametersDict']['Site'] = site
self.queueDict[queueName]['ParametersDict']['GridEnv'] = self.gridEnv
self.queueDict[queueName]['ParametersDict']['Setup'] = gConfig.getValue( '/DIRAC/Setup', 'unknown' )
# Evaluate the CPU limit of the queue according to the Glue convention
# To Do: should be a utility
if "maxCPUTime" in self.queueDict[queueName]['ParametersDict'] and \
"SI00" in self.queueDict[queueName]['ParametersDict']:
maxCPUTime = float( self.queueDict[queueName]['ParametersDict']['maxCPUTime'] )
# For some sites there are crazy values in the CS
maxCPUTime = max( maxCPUTime, 0 )
maxCPUTime = min( maxCPUTime, 86400 * 12.5 )
si00 = float( self.queueDict[queueName]['ParametersDict']['SI00'] )
queueCPUTime = 60. / 250. * maxCPUTime * si00
self.queueDict[queueName]['ParametersDict']['CPUTime'] = int( queueCPUTime )
queueTags = self.queueDict[queueName]['ParametersDict'].get( 'Tag' )
if queueTags and isinstance( queueTags, basestring ):
queueTags = fromChar( queueTags )
self.queueDict[queueName]['ParametersDict']['Tag'] = queueTags
if ceTags:
if queueTags:
allTags = list( set( ceTags + queueTags ) )
self.queueDict[queueName]['ParametersDict']['Tag'] = allTags
else:
self.queueDict[queueName]['ParametersDict']['Tag'] = ceTags
maxRAM = self.queueDict[queueName]['ParametersDict'].get( 'MaxRAM' )
maxRAM = ceMaxRAM if not maxRAM else maxRAM
if maxRAM:
self.queueDict[queueName]['ParametersDict']['MaxRAM'] = maxRAM
if pilotRunDirectory:
self.queueDict[queueName]['ParametersDict']['JobExecDir'] = pilotRunDirectory
qwDir = os.path.join( self.workingDirectory, queue )
mkDir(qwDir)
self.queueDict[queueName]['ParametersDict']['WorkingDirectory'] = qwDir
platform = ''
if "Platform" in self.queueDict[queueName]['ParametersDict']:
platform = self.queueDict[queueName]['ParametersDict']['Platform']
elif "Platform" in ceDict:
platform = ceDict['Platform']
elif "OS" in ceDict:
architecture = ceDict.get( 'architecture', 'x86_64' )
OS = ceDict['OS']
platform = '_'.join( [architecture, OS] )
if platform and not platform in self.platforms:
self.platforms.append( platform )
if not "Platform" in self.queueDict[queueName]['ParametersDict'] and platform:
result = Resources.getDIRACPlatform( platform )
if result['OK']:
self.queueDict[queueName]['ParametersDict']['Platform'] = result['Value'][0]
ceQueueDict = dict( ceDict )
ceQueueDict.update( self.queueDict[queueName]['ParametersDict'] )
# Generate the CE object for the queue or pick the already existing one
# if the queue definition did not change
queueHash = self.__generateQueueHash( ceQueueDict )
if queueName in self.queueCECache and self.queueCECache[queueName]['Hash'] == queueHash:
queueCE = self.queueCECache[queueName]['CE']
else:
result = ceFactory.getCE( ceName = ce,
ceType = ceDict['CEType'],
ceParametersDict = ceQueueDict )
if not result['OK']:
return result
self.queueCECache.setdefault( queueName, {} )
self.queueCECache[queueName]['Hash'] = queueHash
self.queueCECache[queueName]['CE'] = result['Value']
queueCE = self.queueCECache[queueName]['CE']
self.queueDict[queueName]['CE'] = queueCE
self.queueDict[queueName]['CEName'] = ce
self.queueDict[queueName]['CEType'] = ceDict['CEType']
self.queueDict[queueName]['Site'] = site
self.queueDict[queueName]['QueueName'] = queue
self.queueDict[queueName]['Platform'] = platform
result = self.queueDict[queueName]['CE'].isValid()
if not result['OK']:
self.log.fatal( result['Message'] )
return result
if 'BundleProxy' in self.queueDict[queueName]['ParametersDict']:
if self.queueDict[queueName]['ParametersDict']['BundleProxy'].lower() in ['true','yes','1']:
self.queueDict[queueName]['BundleProxy'] = True
elif 'BundleProxy' in ceDict:
if ceDict['BundleProxy'].lower() in ['true','yes','1']:
self.queueDict[queueName]['BundleProxy'] = True
if site not in self.sites:
self.sites.append( site )
return S_OK()
def getQueues(self, resourceDict):
""" Get the list of relevant CEs and their descriptions
"""
self.queueDict = {}
ceFactory = ComputingElementFactory()
for site in resourceDict:
for ce in resourceDict[site]:
ceDict = resourceDict[site][ce]
qDict = ceDict.pop("Queues")
for queue in qDict:
queueName = "%s_%s" % (ce, queue)
self.queueDict[queueName] = {}
self.queueDict[queueName]["ParametersDict"] = qDict[queue]
self.queueDict[queueName]["ParametersDict"]["Queue"] = queue
self.queueDict[queueName]["ParametersDict"]["Site"] = site
self.queueDict[queueName]["ParametersDict"]["GridEnv"] = self.gridEnv
self.queueDict[queueName]["ParametersDict"]["Setup"] = gConfig.getValue("/DIRAC/Setup", "unknown")
# Evaluate the CPU limit of the queue according to the Glue convention
# To Do: should be a utility
if (
"maxCPUTime" in self.queueDict[queueName]["ParametersDict"]
and "SI00" in self.queueDict[queueName]["ParametersDict"]
):
maxCPUTime = float(self.queueDict[queueName]["ParametersDict"]["maxCPUTime"])
# For some sites there are crazy values in the CS
maxCPUTime = max(maxCPUTime, 0)
maxCPUTime = min(maxCPUTime, 86400 * 12.5)
si00 = float(self.queueDict[queueName]["ParametersDict"]["SI00"])
queueCPUTime = 60.0 / 250.0 * maxCPUTime * si00
self.queueDict[queueName]["ParametersDict"]["CPUTime"] = int(queueCPUTime)
qwDir = os.path.join(self.workingDirectory, queue)
if not os.path.exists(qwDir):
os.makedirs(qwDir)
self.queueDict[queueName]["ParametersDict"]["WorkingDirectory"] = qwDir
platform = ""
if "Platform" in self.queueDict[queueName]["ParametersDict"]:
platform = self.queueDict[queueName]["ParametersDict"]["Platform"]
elif "Platform" in ceDict:
platform = ceDict["Platform"]
elif "OS" in ceDict:
architecture = ceDict.get("architecture", "x86_64")
OS = ceDict["OS"]
platform = "_".join([architecture, OS])
if platform and not platform in self.platforms:
self.platforms.append(platform)
if not "Platform" in self.queueDict[queueName]["ParametersDict"] and platform:
result = Resources.getDIRACPlatform(platform)
if result["OK"]:
self.queueDict[queueName]["ParametersDict"]["Platform"] = result["Value"]
ceQueueDict = dict(ceDict)
ceQueueDict.update(self.queueDict[queueName]["ParametersDict"])
result = ceFactory.getCE(ceName=ce, ceType=ceDict["CEType"], ceParametersDict=ceQueueDict)
if not result["OK"]:
return result
self.queueDict[queueName]["CE"] = result["Value"]
self.queueDict[queueName]["CEName"] = ce
self.queueDict[queueName]["CEType"] = ceDict["CEType"]
self.queueDict[queueName]["Site"] = site
self.queueDict[queueName]["QueueName"] = queue
result = self.queueDict[queueName]["CE"].isValid()
if not result["OK"]:
self.log.fatal(result["Message"])
return result
if "BundleProxy" in self.queueDict[queueName]["ParametersDict"]:
self.queueDict[queueName]["BundleProxy"] = True
elif "BundleProxy" in ceDict:
self.queueDict[queueName]["BundleProxy"] = True
if site not in self.sites:
self.sites.append(site)
return S_OK()
