def getQueues( self, resourceDict ): """ Get the list of relevant CEs and their descriptions """ self.queueDict = {} ceFactory = ComputingElementFactory() for site in resourceDict: for ce in resourceDict[site]: ceDict = resourceDict[site][ce] ceTags = ceDict.get( 'Tag' ) if isinstance( ceTags, basestring ): ceTags = fromChar( ceTags ) qDict = ceDict.pop( 'Queues' ) for queue in qDict: queueName = '%s_%s' % ( ce, queue ) self.queueDict[queueName] = {} self.queueDict[queueName]['ParametersDict'] = qDict[queue] self.queueDict[queueName]['ParametersDict']['Queue'] = queue self.queueDict[queueName]['ParametersDict']['Site'] = site self.queueDict[queueName]['ParametersDict']['GridEnv'] = self.gridEnv self.queueDict[queueName]['ParametersDict']['Setup'] = gConfig.getValue( '/DIRAC/Setup', 'unknown' ) # Evaluate the CPU limit of the queue according to the Glue convention # To Do: should be a utility if "maxCPUTime" in self.queueDict[queueName]['ParametersDict'] and \ "SI00" in self.queueDict[queueName]['ParametersDict']: maxCPUTime = float( self.queueDict[queueName]['ParametersDict']['maxCPUTime'] ) # For some sites there are crazy values in the CS maxCPUTime = max( maxCPUTime, 0 ) maxCPUTime = min( maxCPUTime, 86400 * 12.5 ) si00 = float( self.queueDict[queueName]['ParametersDict']['SI00'] ) queueCPUTime = 60. / 250. * maxCPUTime * si00 self.queueDict[queueName]['ParametersDict']['CPUTime'] = int( queueCPUTime ) queueTags = self.queueDict[queueName]['ParametersDict'].get( 'Tag' ) if queueTags and isinstance( queueTags, basestring ): queueTags = fromChar( queueTags ) self.queueDict[queueName]['ParametersDict']['Tag'] = queueTags if ceTags: if queueTags: allTags = list( set( ceTags + queueTags ) ) self.queueDict[queueName]['ParametersDict']['Tag'] = allTags else: self.queueDict[queueName]['ParametersDict']['Tag'] = ceTags maxMemory = self.queueDict[queueName]['ParametersDict'].get( 'MaxRAM', None ) if maxMemory: # MaxRAM value is supposed to be in MB maxMemoryList = range( 1, int( maxMemory )/1000 + 1 ) memoryTags = [ '%dGB' % mem for mem in maxMemoryList ] if memoryTags: self.queueDict[queueName]['ParametersDict'].setdefault( 'Tag', [] ) self.queueDict[queueName]['ParametersDict']['Tag'] += memoryTags qwDir = os.path.join( self.workingDirectory, queue ) if not os.path.exists( qwDir ): os.makedirs( qwDir ) self.queueDict[queueName]['ParametersDict']['WorkingDirectory'] = qwDir platform = '' if "Platform" in self.queueDict[queueName]['ParametersDict']: platform = self.queueDict[queueName]['ParametersDict']['Platform'] elif "Platform" in ceDict: platform = ceDict['Platform'] elif "OS" in ceDict: architecture = ceDict.get( 'architecture', 'x86_64' ) OS = ceDict['OS'] platform = '_'.join( [architecture, OS] ) if platform and not platform in self.platforms: self.platforms.append( platform ) if not "Platform" in self.queueDict[queueName]['ParametersDict'] and platform: result = Resources.getDIRACPlatform( platform ) if result['OK']: self.queueDict[queueName]['ParametersDict']['Platform'] = result['Value'][0] ceQueueDict = dict( ceDict ) ceQueueDict.update( self.queueDict[queueName]['ParametersDict'] ) # Generate the CE object for the queue or pick the already existing one # if the queue definition did not change queueHash = self.__generateQueueHash( ceQueueDict ) if queueName in self.queueCECache and self.queueCECache[queueName]['Hash'] == queueHash: queueCE = self.queueCECache[queueName]['CE'] else: result = ceFactory.getCE( ceName = ce, ceType = ceDict['CEType'], ceParametersDict = ceQueueDict ) if not result['OK']: return result self.queueCECache.setdefault( queueName, {} ) self.queueCECache[queueName]['Hash'] = queueHash self.queueCECache[queueName]['CE'] = result['Value'] queueCE = self.queueCECache[queueName]['CE'] self.queueDict[queueName]['CE'] = queueCE self.queueDict[queueName]['CEName'] = ce self.queueDict[queueName]['CEType'] = ceDict['CEType'] self.queueDict[queueName]['Site'] = site self.queueDict[queueName]['QueueName'] = queue self.queueDict[queueName]['Platform'] = platform result = self.queueDict[queueName]['CE'].isValid() if not result['OK']: self.log.fatal( result['Message'] ) return result if 'BundleProxy' in self.queueDict[queueName]['ParametersDict']: if self.queueDict[queueName]['ParametersDict']['BundleProxy'].lower() in ['true','yes','1']: self.queueDict[queueName]['BundleProxy'] = True elif 'BundleProxy' in ceDict: if ceDict['BundleProxy'].lower() in ['true','yes','1']: self.queueDict[queueName]['BundleProxy'] = True if site not in self.sites: self.sites.append( site ) return S_OK()
def getQueues( self, resourceDict ): """ Get the list of relevant CEs and their descriptions """ self.queueDict = {} ceFactory = ComputingElementFactory() for site in resourceDict: result = self._resources.getSiteFullName( site ) if not result['OK']: continue siteFullName = result['Value'] for ce in resourceDict[site]: ceDict = resourceDict[site][ce] qDict = ceDict.pop( 'Queues' ) for queue in qDict: queueName = '%s_%s' % ( ce, queue ) self.queueDict[queueName] = {} self.queueDict[queueName]['ParametersDict'] = qDict[queue] self.queueDict[queueName]['ParametersDict']['Queue'] = queue self.queueDict[queueName]['ParametersDict']['Site'] = siteFullName self.queueDict[queueName]['ParametersDict']['GridEnv'] = self.gridEnv self.queueDict[queueName]['ParametersDict']['Setup'] = gConfig.getValue( '/DIRAC/Setup', 'unknown' ) # Evaluate the CPU limit of the queue according to the Glue convention # To Do: should be a utility if "maxCPUTime" in self.queueDict[queueName]['ParametersDict'] and \ "SI00" in self.queueDict[queueName]['ParametersDict']: maxCPUTime = float( self.queueDict[queueName]['ParametersDict']['maxCPUTime'] ) # For some sites there are crazy values in the CS maxCPUTime = max( maxCPUTime, 0 ) maxCPUTime = min( maxCPUTime, 86400 * 12.5 ) si00 = float( self.queueDict[queueName]['ParametersDict']['SI00'] ) queueCPUTime = 60. / 250. * maxCPUTime * si00 self.queueDict[queueName]['ParametersDict']['CPUTime'] = int( queueCPUTime ) qwDir = os.path.join( self.workingDirectory, queue ) if not os.path.exists( qwDir ): os.makedirs( qwDir ) self.queueDict[queueName]['ParametersDict']['WorkingDirectory'] = qwDir platform = '' if "Platform" in self.queueDict[queueName]['ParametersDict']: platform = self.queueDict[queueName]['ParametersDict']['Platform'] elif "Platform" in ceDict: platform = ceDict['Platform'] elif "OS" in ceDict: architecture = ceDict.get( 'architecture', 'x86_64' ) OS = ceDict['OS'] platform = '_'.join( [architecture, OS] ) if platform and not platform in self.platforms: self.platforms.append( platform ) if not "Platform" in self.queueDict[queueName]['ParametersDict'] and platform: result = Resources.getDIRACPlatform( platform ) if result['OK']: self.queueDict[queueName]['ParametersDict']['Platform'] = result['Value'] ceQueueDict = dict( ceDict ) ceQueueDict.update( self.queueDict[queueName]['ParametersDict'] ) result = ceFactory.getCE( ceName = ce, ceType = ceDict['CEType'], ceParametersDict = ceQueueDict ) if not result['OK']: return result self.queueDict[queueName]['CE'] = result['Value'] self.queueDict[queueName]['CEName'] = ce self.queueDict[queueName]['CEType'] = ceDict['CEType'] self.queueDict[queueName]['Site'] = siteFullName self.queueDict[queueName]['QueueName'] = queue self.queueDict[queueName]['Platform'] = platform result = self.queueDict[queueName]['CE'].isValid() if not result['OK']: self.log.fatal( result['Message'] ) return result if 'BundleProxy' in self.queueDict[queueName]['ParametersDict']: self.queueDict[queueName]['BundleProxy'] = True elif 'BundleProxy' in ceDict: self.queueDict[queueName]['BundleProxy'] = True if siteFullName not in self.sites: self.sites.append( siteFullName ) return S_OK()
def getQueues(self, resourceDict): """ Get the list of relevant CEs and their descriptions """ self.queueDict = {} ceFactory = ComputingElementFactory() for site in resourceDict: for ce in resourceDict[site]: ceDict = resourceDict[site][ce] qDict = ceDict.pop('Queues') for queue in qDict: queueName = '%s_%s' % (ce, queue) self.queueDict[queueName] = {} self.queueDict[queueName]['ParametersDict'] = qDict[queue] self.queueDict[queueName]['ParametersDict'][ 'Queue'] = queue self.queueDict[queueName]['ParametersDict']['Site'] = site self.queueDict[queueName]['ParametersDict'][ 'GridEnv'] = self.gridEnv self.queueDict[queueName]['ParametersDict'][ 'Setup'] = gConfig.getValue('/DIRAC/Setup', 'unknown') # Evaluate the CPU limit of the queue according to the Glue convention # To Do: should be a utility if "maxCPUTime" in self.queueDict[queueName]['ParametersDict'] and \ "SI00" in self.queueDict[queueName]['ParametersDict']: maxCPUTime = float(self.queueDict[queueName] ['ParametersDict']['maxCPUTime']) # For some sites there are crazy values in the CS maxCPUTime = max(maxCPUTime, 0) maxCPUTime = min(maxCPUTime, 86400 * 12.5) si00 = float(self.queueDict[queueName] ['ParametersDict']['SI00']) queueCPUTime = 60. / 250. * maxCPUTime * si00 self.queueDict[queueName]['ParametersDict'][ 'CPUTime'] = int(queueCPUTime) qwDir = os.path.join(self.workingDirectory, queue) if not os.path.exists(qwDir): os.makedirs(qwDir) self.queueDict[queueName]['ParametersDict'][ 'WorkingDirectory'] = qwDir platform = '' if "Platform" in self.queueDict[queueName][ 'ParametersDict']: platform = self.queueDict[queueName]['ParametersDict'][ 'Platform'] elif "Platform" in ceDict: platform = ceDict['Platform'] elif "OS" in ceDict: architecture = ceDict.get('architecture', 'x86_64') OS = ceDict['OS'] platform = '_'.join([architecture, OS]) if platform and not platform in self.platforms: self.platforms.append(platform) if not "Platform" in self.queueDict[queueName][ 'ParametersDict'] and platform: result = Resources.getDIRACPlatform(platform) if result['OK']: self.queueDict[queueName]['ParametersDict'][ 'Platform'] = result['Value'] ceQueueDict = dict(ceDict) ceQueueDict.update( self.queueDict[queueName]['ParametersDict']) result = ceFactory.getCE(ceName=ce, ceType=ceDict['CEType'], ceParametersDict=ceQueueDict) if not result['OK']: return result self.queueDict[queueName]['CE'] = result['Value'] self.queueDict[queueName]['CEName'] = ce self.queueDict[queueName]['CEType'] = ceDict['CEType'] self.queueDict[queueName]['Site'] = site self.queueDict[queueName]['QueueName'] = queue result = self.queueDict[queueName]['CE'].isValid() if not result['OK']: self.log.fatal(result['Message']) return result if 'BundleProxy' in self.queueDict[queueName][ 'ParametersDict']: self.queueDict[queueName]['BundleProxy'] = True elif 'BundleProxy' in ceDict: self.queueDict[queueName]['BundleProxy'] = True if site not in self.sites: self.sites.append(site) return S_OK()
def getEndpoints(self, resourceDict): """ Get the list of relevant CEs and their descriptions """ self.vmTypeDict = {} ceFactory = EndpointFactory() result = getPilotBootstrapParameters(vo=self.vo, runningPod=self.runningPod) if not result['OK']: return result opParameters = result['Value'] for site in resourceDict: for ce in resourceDict[site]: ceDict = resourceDict[site][ce] ceTags = ceDict.get('Tag', []) if isinstance(ceTags, basestring): ceTags = fromChar(ceTags) ceMaxRAM = ceDict.get('MaxRAM', None) qDict = ceDict.pop('VMTypes') for vmType in qDict: vmTypeName = '%s_%s' % (ce, vmType) self.vmTypeDict[vmTypeName] = {} self.vmTypeDict[vmTypeName]['ParametersDict'] = qDict[ vmType] self.vmTypeDict[vmTypeName]['ParametersDict'][ 'VMType'] = vmType self.vmTypeDict[vmTypeName]['ParametersDict'][ 'Site'] = site self.vmTypeDict[vmTypeName]['ParametersDict'][ 'Setup'] = gConfig.getValue('/DIRAC/Setup', 'unknown') self.vmTypeDict[vmTypeName]['ParametersDict'][ 'CPUTime'] = 99999999 vmTypeTags = self.vmTypeDict[vmTypeName][ 'ParametersDict'].get('Tag') if vmTypeTags and isinstance(vmTypeTags, basestring): vmTypeTags = fromChar(vmTypeTags) self.vmTypeDict[vmTypeName]['ParametersDict'][ 'Tag'] = vmTypeTags if ceTags: if vmTypeTags: allTags = list(set(ceTags + vmTypeTags)) self.vmTypeDict[vmTypeName]['ParametersDict'][ 'Tag'] = allTags else: self.vmTypeDict[vmTypeName]['ParametersDict'][ 'Tag'] = ceTags maxRAM = self.vmTypeDict[vmTypeName]['ParametersDict'].get( 'MaxRAM') maxRAM = ceMaxRAM if not maxRAM else maxRAM if maxRAM: self.vmTypeDict[vmTypeName]['ParametersDict'][ 'MaxRAM'] = maxRAM ceWholeNode = ceDict.get('WholeNode', 'true') wholeNode = self.vmTypeDict[vmTypeName][ 'ParametersDict'].get('WholeNode', ceWholeNode) if wholeNode.lower() in ('yes', 'true'): self.vmTypeDict[vmTypeName][ 'ParametersDict'].setdefault('Tag', []) self.vmTypeDict[vmTypeName]['ParametersDict'][ 'Tag'].append('WholeNode') platform = '' if "Platform" in self.vmTypeDict[vmTypeName][ 'ParametersDict']: platform = self.vmTypeDict[vmTypeName][ 'ParametersDict']['Platform'] elif "Platform" in ceDict: platform = ceDict['Platform'] if platform and platform not in self.platforms: self.platforms.append(platform) if "Platform" not in self.vmTypeDict[vmTypeName][ 'ParametersDict'] and platform: result = Resources.getDIRACPlatform(platform) if result['OK']: self.vmTypeDict[vmTypeName]['ParametersDict'][ 'Platform'] = result['Value'][0] ceVMTypeDict = dict(ceDict) ceVMTypeDict['CEName'] = ce ceVMTypeDict['VO'] = self.vo ceVMTypeDict['VMType'] = vmType ceVMTypeDict['RunningPod'] = self.runningPod ceVMTypeDict['CSServers'] = gConfig.getValue( "/DIRAC/Configuration/Servers", []) ceVMTypeDict.update( self.vmTypeDict[vmTypeName]['ParametersDict']) # Allow a resource-specifc CAPath to be set (as some clouds have their own CAs) # Otherwise fall back to the system-wide default(s) if 'CAPath' not in ceVMTypeDict: ceVMTypeDict['CAPath'] = gConfig.getValue( '/DIRAC/Security/CAPath', "/opt/dirac/etc/grid-security/certificates/cas.pem" ) # Generate the CE object for the vmType or pick the already existing one # if the vmType definition did not change vmTypeHash = self.__generateVMTypeHash(ceVMTypeDict) if vmTypeName in self.vmTypeCECache and self.vmTypeCECache[ vmTypeName]['Hash'] == vmTypeHash: vmTypeCE = self.vmTypeCECache[vmTypeName]['CE'] else: result = ceFactory.getCEObject(parameters=ceVMTypeDict) if not result['OK']: return result self.vmTypeCECache.setdefault(vmTypeName, {}) self.vmTypeCECache[vmTypeName]['Hash'] = vmTypeHash self.vmTypeCECache[vmTypeName]['CE'] = result['Value'] vmTypeCE = self.vmTypeCECache[vmTypeName]['CE'] vmTypeCE.setBootstrapParameters(opParameters) self.vmTypeDict[vmTypeName]['CE'] = vmTypeCE self.vmTypeDict[vmTypeName]['CEName'] = ce self.vmTypeDict[vmTypeName]['CEType'] = ceDict['CEType'] self.vmTypeDict[vmTypeName]['Site'] = site self.vmTypeDict[vmTypeName]['VMType'] = vmType self.vmTypeDict[vmTypeName]['Platform'] = platform self.vmTypeDict[vmTypeName]['MaxInstances'] = ceDict[ 'MaxInstances'] if not self.vmTypeDict[vmTypeName]['CE'].isValid(): self.log.error( 'Failed to instantiate CloudEndpoint for %s' % vmTypeName) continue if site not in self.sites: self.sites.append(site) return S_OK()
def getEndpoints(self, resourceDict): """Get the list of relevant CEs and their descriptions""" self.vmTypeDict = {} ceFactory = EndpointFactory() result = getPilotBootstrapParameters(vo=self.vo, runningPod=self.runningPod) if not result["OK"]: return result opParameters = result["Value"] for site in resourceDict: for ce in resourceDict[site]: ceDict = resourceDict[site][ce] ceTags = ceDict.get("Tag", []) if isinstance(ceTags, six.string_types): ceTags = fromChar(ceTags) ceMaxRAM = ceDict.get("MaxRAM", None) qDict = ceDict.pop("VMTypes") for vmType in qDict: vmTypeName = "%s_%s" % (ce, vmType) self.vmTypeDict[vmTypeName] = {} self.vmTypeDict[vmTypeName]["ParametersDict"] = qDict[vmType] self.vmTypeDict[vmTypeName]["ParametersDict"]["VMType"] = vmType self.vmTypeDict[vmTypeName]["ParametersDict"]["Site"] = site self.vmTypeDict[vmTypeName]["ParametersDict"]["Setup"] = gConfig.getValue("/DIRAC/Setup", "unknown") self.vmTypeDict[vmTypeName]["ParametersDict"]["CPUTime"] = 99999999 vmTypeTags = self.vmTypeDict[vmTypeName]["ParametersDict"].get("Tag") if vmTypeTags and isinstance(vmTypeTags, six.string_types): vmTypeTags = fromChar(vmTypeTags) self.vmTypeDict[vmTypeName]["ParametersDict"]["Tag"] = vmTypeTags if ceTags: if vmTypeTags: allTags = list(set(ceTags + vmTypeTags)) self.vmTypeDict[vmTypeName]["ParametersDict"]["Tag"] = allTags else: self.vmTypeDict[vmTypeName]["ParametersDict"]["Tag"] = ceTags maxRAM = self.vmTypeDict[vmTypeName]["ParametersDict"].get("MaxRAM") maxRAM = ceMaxRAM if not maxRAM else maxRAM if maxRAM: self.vmTypeDict[vmTypeName]["ParametersDict"]["MaxRAM"] = maxRAM ceWholeNode = ceDict.get("WholeNode", "true") wholeNode = self.vmTypeDict[vmTypeName]["ParametersDict"].get("WholeNode", ceWholeNode) if wholeNode.lower() in ("yes", "true"): self.vmTypeDict[vmTypeName]["ParametersDict"].setdefault("Tag", []) self.vmTypeDict[vmTypeName]["ParametersDict"]["Tag"].append("WholeNode") platform = "" if "Platform" in self.vmTypeDict[vmTypeName]["ParametersDict"]: platform = self.vmTypeDict[vmTypeName]["ParametersDict"]["Platform"] elif "Platform" in ceDict: platform = ceDict["Platform"] if platform and platform not in self.platforms: self.platforms.append(platform) if "Platform" not in self.vmTypeDict[vmTypeName]["ParametersDict"] and platform: result = Resources.getDIRACPlatform(platform) if result["OK"]: self.vmTypeDict[vmTypeName]["ParametersDict"]["Platform"] = result["Value"][0] ceVMTypeDict = dict(ceDict) ceVMTypeDict["CEName"] = ce ceVMTypeDict["VO"] = self.vo ceVMTypeDict["VMType"] = vmType ceVMTypeDict["RunningPod"] = self.runningPod ceVMTypeDict["CSServers"] = gConfig.getValue("/DIRAC/Configuration/Servers", []) ceVMTypeDict.update(self.vmTypeDict[vmTypeName]["ParametersDict"]) # Allow a resource-specifc CAPath to be set (as some clouds have their own CAs) # Otherwise fall back to the system-wide default(s) if "CAPath" not in ceVMTypeDict: ceVMTypeDict["CAPath"] = gConfig.getValue( "/DIRAC/Security/CAPath", "/opt/dirac/etc/grid-security/certificates/cas.pem" ) # Generate the CE object for the vmType or pick the already existing one # if the vmType definition did not change vmTypeHash = self.__generateVMTypeHash(ceVMTypeDict) if vmTypeName in self.vmTypeCECache and self.vmTypeCECache[vmTypeName]["Hash"] == vmTypeHash: vmTypeCE = self.vmTypeCECache[vmTypeName]["CE"] else: result = ceFactory.getCEObject(parameters=ceVMTypeDict) if not result["OK"]: return result self.vmTypeCECache.setdefault(vmTypeName, {}) self.vmTypeCECache[vmTypeName]["Hash"] = vmTypeHash self.vmTypeCECache[vmTypeName]["CE"] = result["Value"] vmTypeCE = self.vmTypeCECache[vmTypeName]["CE"] vmTypeCE.setBootstrapParameters(opParameters) self.vmTypeDict[vmTypeName]["CE"] = vmTypeCE self.vmTypeDict[vmTypeName]["CEName"] = ce self.vmTypeDict[vmTypeName]["CEType"] = ceDict["CEType"] self.vmTypeDict[vmTypeName]["Site"] = site self.vmTypeDict[vmTypeName]["VMType"] = vmType self.vmTypeDict[vmTypeName]["Platform"] = platform self.vmTypeDict[vmTypeName]["MaxInstances"] = ceDict["MaxInstances"] if not self.vmTypeDict[vmTypeName]["CE"].isValid(): self.log.error("Failed to instantiate CloudEndpoint for %s" % vmTypeName) continue if site not in self.sites: self.sites.append(site) return S_OK()
def getImages(self, resourceDict): """ Get the list of relevant CEs and their descriptions """ self.imageDict = {} ceFactory = EndpointFactory() result = getPilotBootstrapParameters(vo=self.vo, runningPod=self.runningPod) if not result['OK']: return result opParameters = result['Value'] for site in resourceDict: for ce in resourceDict[site]: ceDict = resourceDict[site][ce] ceTags = ceDict.get('Tag', []) if isinstance(ceTags, basestring): ceTags = fromChar(ceTags) ceMaxRAM = ceDict.get('MaxRAM', None) qDict = ceDict.pop('Images') for image in qDict: imageName = '%s_%s' % (ce, image) self.imageDict[imageName] = {} self.imageDict[imageName]['ParametersDict'] = qDict[image] self.imageDict[imageName]['ParametersDict'][ 'Image'] = image self.imageDict[imageName]['ParametersDict']['Site'] = site self.imageDict[imageName]['ParametersDict'][ 'Setup'] = gConfig.getValue('/DIRAC/Setup', 'unknown') self.imageDict[imageName]['ParametersDict'][ 'CPUTime'] = 99999999 imageTags = self.imageDict[imageName][ 'ParametersDict'].get('Tag') if imageTags and isinstance(imageTags, basestring): imageTags = fromChar(imageTags) self.imageDict[imageName]['ParametersDict'][ 'Tag'] = imageTags if ceTags: if imageTags: allTags = list(set(ceTags + imageTags)) self.imageDict[imageName]['ParametersDict'][ 'Tag'] = allTags else: self.imageDict[imageName]['ParametersDict'][ 'Tag'] = ceTags maxRAM = self.imageDict[imageName]['ParametersDict'].get( 'MaxRAM') maxRAM = ceMaxRAM if not maxRAM else maxRAM if maxRAM: self.imageDict[imageName]['ParametersDict'][ 'MaxRAM'] = maxRAM platform = '' if "Platform" in self.imageDict[imageName][ 'ParametersDict']: platform = self.imageDict[imageName]['ParametersDict'][ 'Platform'] elif "Platform" in ceDict: platform = ceDict['Platform'] if platform and not platform in self.platforms: self.platforms.append(platform) if not "Platform" in self.imageDict[imageName][ 'ParametersDict'] and platform: result = Resources.getDIRACPlatform(platform) if result['OK']: self.imageDict[imageName]['ParametersDict'][ 'Platform'] = result['Value'][0] ceImageDict = dict(ceDict) ceImageDict['CEName'] = ce ceImageDict['VO'] = self.vo ceImageDict['Image'] = image ceImageDict['RunningPod'] = self.runningPod ceImageDict['CSServers'] = gConfig.getValue( "/DIRAC/Configuration/Servers", []) ceImageDict.update( self.imageDict[imageName]['ParametersDict']) ceImageDict.update(opParameters) # Generate the CE object for the image or pick the already existing one # if the image definition did not change imageHash = self.__generateImageHash(ceImageDict) if imageName in self.imageCECache and self.imageCECache[ imageName]['Hash'] == imageHash: imageCE = self.imageCECache[imageName]['CE'] else: result = ceFactory.getCEObject(parameters=ceImageDict) if not result['OK']: return result self.imageCECache.setdefault(imageName, {}) self.imageCECache[imageName]['Hash'] = imageHash self.imageCECache[imageName]['CE'] = result['Value'] imageCE = self.imageCECache[imageName]['CE'] self.imageDict[imageName]['CE'] = imageCE self.imageDict[imageName]['CEName'] = ce self.imageDict[imageName]['CEType'] = ceDict['CEType'] self.imageDict[imageName]['Site'] = site self.imageDict[imageName]['ImageName'] = image self.imageDict[imageName]['Platform'] = platform self.imageDict[imageName]['MaxInstances'] = ceDict[ 'MaxInstances'] if not self.imageDict[imageName]['CE'].isValid(): self.log.error( 'Failed to instantiate CloudEndpoint for %s' % imageName) continue if site not in self.sites: self.sites.append(site) return S_OK()
def getQueues( self, resourceDict ): """ Get the list of relevant CEs and their descriptions """ self.queueDict = {} ceFactory = ComputingElementFactory() for site in resourceDict: for ce in resourceDict[site]: ceDict = resourceDict[site][ce] ceTags = ceDict.get( 'Tag', [] ) pilotRunDirectory = ceDict.get( 'PilotRunDirectory', '' ) if isinstance( ceTags, basestring ): ceTags = fromChar( ceTags ) ceMaxRAM = ceDict.get( 'MaxRAM', None ) qDict = ceDict.pop( 'Queues' ) for queue in qDict: queueName = '%s_%s' % ( ce, queue ) self.queueDict[queueName] = {} self.queueDict[queueName]['ParametersDict'] = qDict[queue] self.queueDict[queueName]['ParametersDict']['Queue'] = queue self.queueDict[queueName]['ParametersDict']['Site'] = site self.queueDict[queueName]['ParametersDict']['GridEnv'] = self.gridEnv self.queueDict[queueName]['ParametersDict']['Setup'] = gConfig.getValue( '/DIRAC/Setup', 'unknown' ) # Evaluate the CPU limit of the queue according to the Glue convention # To Do: should be a utility if "maxCPUTime" in self.queueDict[queueName]['ParametersDict'] and \ "SI00" in self.queueDict[queueName]['ParametersDict']: maxCPUTime = float( self.queueDict[queueName]['ParametersDict']['maxCPUTime'] ) # For some sites there are crazy values in the CS maxCPUTime = max( maxCPUTime, 0 ) maxCPUTime = min( maxCPUTime, 86400 * 12.5 ) si00 = float( self.queueDict[queueName]['ParametersDict']['SI00'] ) queueCPUTime = 60. / 250. * maxCPUTime * si00 self.queueDict[queueName]['ParametersDict']['CPUTime'] = int( queueCPUTime ) queueTags = self.queueDict[queueName]['ParametersDict'].get( 'Tag' ) if queueTags and isinstance( queueTags, basestring ): queueTags = fromChar( queueTags ) self.queueDict[queueName]['ParametersDict']['Tag'] = queueTags if ceTags: if queueTags: allTags = list( set( ceTags + queueTags ) ) self.queueDict[queueName]['ParametersDict']['Tag'] = allTags else: self.queueDict[queueName]['ParametersDict']['Tag'] = ceTags maxRAM = self.queueDict[queueName]['ParametersDict'].get( 'MaxRAM' ) maxRAM = ceMaxRAM if not maxRAM else maxRAM if maxRAM: self.queueDict[queueName]['ParametersDict']['MaxRAM'] = maxRAM if pilotRunDirectory: self.queueDict[queueName]['ParametersDict']['JobExecDir'] = pilotRunDirectory qwDir = os.path.join( self.workingDirectory, queue ) mkDir(qwDir) self.queueDict[queueName]['ParametersDict']['WorkingDirectory'] = qwDir platform = '' if "Platform" in self.queueDict[queueName]['ParametersDict']: platform = self.queueDict[queueName]['ParametersDict']['Platform'] elif "Platform" in ceDict: platform = ceDict['Platform'] elif "OS" in ceDict: architecture = ceDict.get( 'architecture', 'x86_64' ) OS = ceDict['OS'] platform = '_'.join( [architecture, OS] ) if platform and not platform in self.platforms: self.platforms.append( platform ) if not "Platform" in self.queueDict[queueName]['ParametersDict'] and platform: result = Resources.getDIRACPlatform( platform ) if result['OK']: self.queueDict[queueName]['ParametersDict']['Platform'] = result['Value'][0] ceQueueDict = dict( ceDict ) ceQueueDict.update( self.queueDict[queueName]['ParametersDict'] ) # Generate the CE object for the queue or pick the already existing one # if the queue definition did not change queueHash = self.__generateQueueHash( ceQueueDict ) if queueName in self.queueCECache and self.queueCECache[queueName]['Hash'] == queueHash: queueCE = self.queueCECache[queueName]['CE'] else: result = ceFactory.getCE( ceName = ce, ceType = ceDict['CEType'], ceParametersDict = ceQueueDict ) if not result['OK']: return result self.queueCECache.setdefault( queueName, {} ) self.queueCECache[queueName]['Hash'] = queueHash self.queueCECache[queueName]['CE'] = result['Value'] queueCE = self.queueCECache[queueName]['CE'] self.queueDict[queueName]['CE'] = queueCE self.queueDict[queueName]['CEName'] = ce self.queueDict[queueName]['CEType'] = ceDict['CEType'] self.queueDict[queueName]['Site'] = site self.queueDict[queueName]['QueueName'] = queue self.queueDict[queueName]['Platform'] = platform result = self.queueDict[queueName]['CE'].isValid() if not result['OK']: self.log.fatal( result['Message'] ) return result if 'BundleProxy' in self.queueDict[queueName]['ParametersDict']: if self.queueDict[queueName]['ParametersDict']['BundleProxy'].lower() in ['true','yes','1']: self.queueDict[queueName]['BundleProxy'] = True elif 'BundleProxy' in ceDict: if ceDict['BundleProxy'].lower() in ['true','yes','1']: self.queueDict[queueName]['BundleProxy'] = True if site not in self.sites: self.sites.append( site ) return S_OK()
def getQueues(self, resourceDict): """ Get the list of relevant CEs and their descriptions """ self.queueDict = {} ceFactory = ComputingElementFactory() for site in resourceDict: for ce in resourceDict[site]: ceDict = resourceDict[site][ce] qDict = ceDict.pop("Queues") for queue in qDict: queueName = "%s_%s" % (ce, queue) self.queueDict[queueName] = {} self.queueDict[queueName]["ParametersDict"] = qDict[queue] self.queueDict[queueName]["ParametersDict"]["Queue"] = queue self.queueDict[queueName]["ParametersDict"]["Site"] = site self.queueDict[queueName]["ParametersDict"]["GridEnv"] = self.gridEnv self.queueDict[queueName]["ParametersDict"]["Setup"] = gConfig.getValue("/DIRAC/Setup", "unknown") # Evaluate the CPU limit of the queue according to the Glue convention # To Do: should be a utility if ( "maxCPUTime" in self.queueDict[queueName]["ParametersDict"] and "SI00" in self.queueDict[queueName]["ParametersDict"] ): maxCPUTime = float(self.queueDict[queueName]["ParametersDict"]["maxCPUTime"]) # For some sites there are crazy values in the CS maxCPUTime = max(maxCPUTime, 0) maxCPUTime = min(maxCPUTime, 86400 * 12.5) si00 = float(self.queueDict[queueName]["ParametersDict"]["SI00"]) queueCPUTime = 60.0 / 250.0 * maxCPUTime * si00 self.queueDict[queueName]["ParametersDict"]["CPUTime"] = int(queueCPUTime) qwDir = os.path.join(self.workingDirectory, queue) if not os.path.exists(qwDir): os.makedirs(qwDir) self.queueDict[queueName]["ParametersDict"]["WorkingDirectory"] = qwDir platform = "" if "Platform" in self.queueDict[queueName]["ParametersDict"]: platform = self.queueDict[queueName]["ParametersDict"]["Platform"] elif "Platform" in ceDict: platform = ceDict["Platform"] elif "OS" in ceDict: architecture = ceDict.get("architecture", "x86_64") OS = ceDict["OS"] platform = "_".join([architecture, OS]) if platform and not platform in self.platforms: self.platforms.append(platform) if not "Platform" in self.queueDict[queueName]["ParametersDict"] and platform: result = Resources.getDIRACPlatform(platform) if result["OK"]: self.queueDict[queueName]["ParametersDict"]["Platform"] = result["Value"] ceQueueDict = dict(ceDict) ceQueueDict.update(self.queueDict[queueName]["ParametersDict"]) result = ceFactory.getCE(ceName=ce, ceType=ceDict["CEType"], ceParametersDict=ceQueueDict) if not result["OK"]: return result self.queueDict[queueName]["CE"] = result["Value"] self.queueDict[queueName]["CEName"] = ce self.queueDict[queueName]["CEType"] = ceDict["CEType"] self.queueDict[queueName]["Site"] = site self.queueDict[queueName]["QueueName"] = queue result = self.queueDict[queueName]["CE"].isValid() if not result["OK"]: self.log.fatal(result["Message"]) return result if "BundleProxy" in self.queueDict[queueName]["ParametersDict"]: self.queueDict[queueName]["BundleProxy"] = True elif "BundleProxy" in ceDict: self.queueDict[queueName]["BundleProxy"] = True if site not in self.sites: self.sites.append(site) return S_OK()