def initialize( self ): # TODO: Have no default and if no mail is found then use the diracAdmin group # and resolve all associated mail addresses. self.addressTo = self.am_getOption( 'MailTo', self.addressTo ) self.addressFrom = self.am_getOption( 'MailFrom', self.addressFrom ) # create a list of alternative bdii urls self.alternativeBDIIs = self.am_getOption( 'AlternativeBDIIs', [] ) # check if the bdii url is appended by a port number, if not append the default 2170 for index, url in enumerate( self.alternativeBDIIs ): if not url.split( ':' )[-1].isdigit(): self.alternativeBDIIs[index] += ':2170' if self.addressTo and self.addressFrom: self.log.info( "MailTo", self.addressTo ) self.log.info( "MailFrom", self.addressFrom ) if self.alternativeBDIIs : self.log.info( "AlternativeBDII URLs:", self.alternativeBDIIs ) self.subject = "CE2CSAgent" # This sets the Default Proxy to used as that defined under # /Operations/Shifter/SAMManager # the shifterProxy option in the Configuration can be used to change this default. self.am_setOption( 'shifterProxy', 'SAMManager' ) self.voName = self.am_getOption( 'VirtualOrganization', self.voName ) if not self.voName: self.voName = getVO() if not self.voName: self.log.fatal( "VO option not defined for agent" ) return S_ERROR() self.csAPI = CSAPI() return self.csAPI.initialize()
def updateCS(changeSet): global vo, dry, ceBdiiDict changeList = sorted(changeSet) if dry: gLogger.notice("The following needed changes are detected:\n") else: gLogger.notice("We are about to make the following changes to CS:\n") for entry in changeList: gLogger.notice("%s/%s %s -> %s" % entry) if not dry: csAPI = CSAPI() csAPI.initialize() result = csAPI.downloadCSData() if not result["OK"]: gLogger.error("Failed to initialize CSAPI object", result["Message"]) DIRACExit(-1) for section, option, value, new_value in changeSet: if value == "Unknown" or not value: csAPI.setOption(cfgPath(section, option), new_value) else: csAPI.modifyValue(cfgPath(section, option), new_value) yn = six.moves.input("Do you want to commit changes to CS ? [default yes] [yes|no]: ") if yn == "" or yn.lower().startswith("y"): result = csAPI.commit() if not result["OK"]: gLogger.error("Error while commit to CS", result["Message"]) else: gLogger.notice("Successfully committed %d changes to CS" % len(changeSet))
class TestComponentInstallation( unittest.TestCase ): """ Contains methods for testing of separate elements """ def setUp( self ): self.host = 'localhost' self.notificationPort = 9154 self.rootPwd = '' self.csClient = CSAPI() self.monitoringClient = ComponentMonitoringClient() self.client = SystemAdministratorClientCLI( self.host ) self.csClient.downloadCSData() result = self.csClient.getCurrentCFG() if not result[ 'OK' ]: raise Exception( result[ 'Message' ] ) cfg = result[ 'Value' ] setup = cfg.getOption( 'DIRAC/Setup', 'JenkinsSetup' ) self.frameworkSetup = cfg.getOption( 'DIRAC/Setups/' + setup + '/Framework' ) self.rootPwd = cfg.getOption( 'Systems/Databases/Password' ) self.diracPwd = self.rootPwd def tearDown( self ): pass
def initialize(self): self.log = gLogger.getSubLogger(self.__class__.__name__) # API initialization is required to get an up-to-date configuration from the CS self.csAPI = CSAPI() self.csAPI.initialize() # temporary buffer for network accounting objects + some parameters self.buffer = { } # { {addTime: datetime.now(), object: Network() }, ... } self.bufferTimeout = self.am_getOption("BufferTimeout", NetworkAgent.BUFFER_TIMEOUT) # internal list of message queue consumers self.consumers = [] # host-to-dirac name dictionary self.nameDictionary = {} # statistics self.messagesCount = 0 # number of received messages self.messagesCountOld = 0 # previous number of received messages (used to check connection status) self.skippedMessagesCount = 0 # number of skipped messages (errors, unsupported metrics, etc.) self.PLRMetricCount = 0 # number of received packet-loss-rate metrics self.OWDMetricCount = 0 # number of received one-way-delay metrics self.skippedMetricCount = 0 # number of skipped metrics (errors, invalid data, etc.) self.insertedCount = 0 # number of properly inserted accounting objects self.removedCount = 0 # number of removed accounting objects (missing data) return S_OK()
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument( "ShifterRole: Name of the shifter role, e.g. DataManager") Script.registerArgument( "UserName: A user name, as registered in Registry section") Script.registerArgument( "DIRACGroup: DIRAC Group, e.g. diracAdmin (the user has to have this role)" ) Script.parseCommandLine(ignoreErrors=True) csAPI = CSAPI() shifterRole, userName, diracGroup = Script.getPositionalArgs(group=True) res = csAPI.addShifter( {shifterRole: { "User": userName, "Group": diracGroup }}) if not res["OK"]: gLogger.error("Could not add shifter", ": " + res["Message"]) DIRACExit(1) res = csAPI.commit() if not res["OK"]: gLogger.error("Could not add shifter", ": " + res["Message"]) DIRACExit(1) gLogger.notice("Added shifter %s as user %s with group %s" % (shifterRole, userName, diracGroup))
def __setCSElementStatus(self, elementName, elementType, statusType, status): """ Sets on the CS the Elements status """ # DIRAC doesn't store the status of ComputingElements nor FTS in the CS, so here we can just do nothing if elementType in ('ComputingElement', 'FTS'): return S_OK() # If we are here it is because elementType is either 'StorageElement' or 'Catalog' statuses = self.rssConfig.getConfigStatusType(elementType) if statusType not in statuses: gLogger.error("%s is not a valid statusType" % statusType) return S_ERROR("%s is not a valid statusType: %s" % (statusType, statuses)) if elementType == 'StorageElement': cs_path = "/Resources/StorageElements" elif elementType == 'Catalog': cs_path = "/Resources/FileCatalogs" # FIXME: This a probably outdated location (new one is in /Operations/[]/Services/Catalogs) # but needs to be VO-aware statusType = 'Status' csAPI = CSAPI() csAPI.setOption("%s/%s/%s/%s" % (cs_path, elementName, elementType, statusType), status) res = csAPI.commitChanges() if not res['OK']: gLogger.warn('CS: %s' % res['Message']) return res
def initialize(self): """ Initialize the default parameters """ voNames = self.am_getOption('VO', []) if not voNames[0].lower() == "none": if voNames[0].lower() == "any": voNames = [] result = getVOMSVOs(voNames) if not result['OK']: return result self.__voDict = result['Value'] self.log.notice("VOs: %s" % self.__voDict.keys()) self.csapi = CSAPI() self.dryRun = self.am_getOption('DryRun', self.dryRun) self.autoAddUsers = self.am_getOption('AutoAddUsers', self.autoAddUsers) self.autoModifyUsers = self.am_getOption('AutoModifyUsers', self.autoModifyUsers) self.autoDeleteUsers = self.am_getOption('AutoDeleteUsers', self.autoDeleteUsers) self.detailedReport = self.am_getOption('DetailedReport', self.detailedReport) self.makeFCEntry = self.am_getOption('MakeHomeDirectory', self.makeFCEntry) return S_OK()
def updateCS( changeSet ): global vo, dry, ceBdiiDict changeList = list( changeSet ) changeList.sort() if dry: gLogger.notice( 'The following needed changes are detected:\n' ) else: gLogger.notice( 'We are about to make the following changes to CS:\n' ) for entry in changeList: gLogger.notice( "%s/%s %s -> %s" % entry ) if not dry: csAPI = CSAPI() csAPI.initialize() result = csAPI.downloadCSData() if not result['OK']: gLogger.error( 'Failed to initialize CSAPI object', result['Message'] ) DIRACExit( -1 ) for section, option, value, new_value in changeSet: if value == 'Unknown' or not value: csAPI.setOption( cfgPath( section, option ), new_value ) else: csAPI.modifyValue( cfgPath( section, option ), new_value ) yn = raw_input( 'Do you want to commit changes to CS ? [default yes] [yes|no]: ' ) if yn == '' or yn.lower().startswith( 'y' ): result = csAPI.commit() if not result['OK']: gLogger.error( "Error while commit to CS", result['Message'] ) else: gLogger.notice( "Successfully committed %d changes to CS" % len( changeSet ) )
def main(): Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() csAPI = CSAPI() if len(args) < 3: Script.showHelp(exitCode=1) shifterRole = args[0] userName = args[1] diracGroup = args[2] res = csAPI.addShifter( {shifterRole: { 'User': userName, 'Group': diracGroup }}) if not res['OK']: gLogger.error("Could not add shifter", ": " + res['Message']) DIRACExit(1) res = csAPI.commit() if not res['OK']: gLogger.error("Could not add shifter", ": " + res['Message']) DIRACExit(1) gLogger.notice("Added shifter %s as user %s with group %s" % (shifterRole, userName, diracGroup))
def __setCSElementStatus(self, elementName, elementType, statusType, status): """ Sets on the CS the Elements status """ # DIRAC doesn't store the status of ComputingElements nor FTS in the CS, so here we can just do nothing if elementType in ('ComputingElement', 'FTS'): return S_OK() # If we are here it is because elementType is either 'StorageElement' or 'Catalog' statuses = self.rssConfig.getConfigStatusType(elementType) if statusType not in statuses: gLogger.error("%s is not a valid statusType" % statusType) return S_ERROR("%s is not a valid statusType: %s" % (statusType, statuses)) if elementType == 'StorageElement': cs_path = "/Resources/StorageElements" elif elementType == 'Catalog': cs_path = "/Resources/FileCatalogs" #FIXME: This a probably outdated location (new one is in /Operations/[]/Services/Catalogs) # but needs to be VO-aware statusType = 'Status' csAPI = CSAPI() csAPI.setOption( "%s/%s/%s/%s" % (cs_path, elementName, elementType, statusType), status) res = csAPI.commitChanges() if not res['OK']: gLogger.warn('CS: %s' % res['Message']) return res
def initialize(self): """Run at the agent initialization (normally every 500 cycles)""" # client to connect to GOCDB self.GOCDBClient = GOCDBClient() self.dryRun = self.am_getOption("DryRun", self.dryRun) # API needed to update configuration stored by CS self.csAPI = CSAPI() return self.csAPI.initialize()
def __init__(self): """initialise.""" CSAPI.__init__(self) self._num_changes = 0 self._append_dict = ConfigDefaultDict(list) self._append_unique_dict = ConfigDefaultDict(set) result = self.initialize() if not result['OK']: gLogger.error('Failed to initialise CSAPI object:', result['Message']) raise RuntimeError(result['Message'])
def __init__(self, vo, autoModifyUsers=True, autoAddUsers=True, autoDeleteUsers=False): self.log = gLogger.getSubLogger("VOMS2CSSynchronizer") self.csapi = CSAPI() self.vo = vo self.vomsVOName = getVOOption(vo, "VOMSName", "") if not self.vomsVOName: raise Exception("VOMS name not defined for VO %s" % vo) self.adminMsgs = {'Errors': [], 'Info': []} self.vomsUserDict = {} self.autoModifyUsers = autoModifyUsers self.autoAddUsers = autoAddUsers self.autoDeleteUsers = autoDeleteUsers
def initialize(self): """ Gets run paramaters from the configuration """ self.addressTo = self.am_getOption('MailTo', self.addressTo) self.addressFrom = self.am_getOption('MailFrom', self.addressFrom) # Create a list of alternative bdii urls self.alternativeBDIIs = self.am_getOption('AlternativeBDIIs', self.alternativeBDIIs) self.host = self.am_getOption('Host', self.host) self.glue2URLs = self.am_getOption('GLUE2URLs', self.glue2URLs) self.glue2Only = self.am_getOption('GLUE2Only', self.glue2Only) # Check if the bdii url is appended by a port number, if not append the default 2170 for index, url in enumerate(self.alternativeBDIIs): if not url.split(':')[-1].isdigit(): self.alternativeBDIIs[index] += ':2170' if self.addressTo and self.addressFrom: self.log.info("MailTo", self.addressTo) self.log.info("MailFrom", self.addressFrom) if self.alternativeBDIIs: self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs) self.processCEs = self.am_getOption('ProcessCEs', self.processCEs) self.processSEs = self.am_getOption('ProcessSEs', self.processSEs) self.selectedSites = self.am_getOption('SelectedSites', []) self.dryRun = self.am_getOption('DryRun', self.dryRun) self.voName = self.am_getOption('VirtualOrganization', self.voName) if not self.voName: self.voName = self.am_getOption('VO', []) if not self.voName or (len(self.voName) == 1 and self.voName[0].lower() == 'all'): # Get all VOs defined in the configuration self.voName = [] result = getVOs() if result['OK']: vos = result['Value'] for vo in vos: vomsVO = getVOOption(vo, "VOMSName") if vomsVO: self.voName.append(vomsVO) if self.voName: self.log.info("Agent will manage VO(s) %s" % self.voName) else: self.log.fatal("VirtualOrganization option not defined for agent") return S_ERROR() self.csAPI = CSAPI() return self.csAPI.initialize()
def __init__(self): """Internal initialization of the DIRAC Admin API. """ super(DiracAdmin, self).__init__() self.csAPI = CSAPI() self.dbg = False if gConfig.getValue(self.section + '/LogLevel', 'DEBUG') == 'DEBUG': self.dbg = True self.scratchDir = gConfig.getValue(self.section + '/ScratchDir', '/tmp') self.currentDir = os.getcwd()
def updateFlag(): # Here now setting the flag as the following inside /Operations/Defaults: # in Operations/Defaults/Services/JobMonitoring/useESForJobParametersFlag from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI csAPI = CSAPI() res = csAPI.createSection('Operations/Defaults/Services/') if not res['OK']: print(res['Message']) exit(1) res = csAPI.createSection('Operations/Defaults/Services/JobMonitoring/') if not res['OK']: print(res['Message']) exit(1) csAPI.setOption( 'Operations/Defaults/Services/JobMonitoring/useESForJobParametersFlag', True) csAPI.commit() # Now we need to restart the services for the new configuration to be picked up time.sleep(2) os.system("dirac-restart-component WorkloadManagement JobMonitoring") os.system("dirac-restart-component WorkloadManagement JobStateUpdate") time.sleep(5)
def __init__(self): from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI self.detmodels = {} self.lcgeo_env = "lcgeo_DIR" self.ddhep_env = "DD4HEP" self.softSec = "/Operations/Defaults/AvailableTarBalls" self.version = '' self.platform = 'x86_64-slc5-gcc43-opt' self.comment = "" self.name = "ddsim" self.csapi = CSAPI() self.tarBallName = None self.md5sum = None
def getMQParamsFromCS(mqURI): """Function gets parameters of a MQ destination (queue/topic) from the CS. Args: mqURI(str):Pseudo URI identifing the MQ service. It has the following format: mqConnection::DestinationType::DestinationName e.g. blabla.cern.ch::Queues::MyQueue1 mType(str): 'consumer' or 'producer' Returns: S_OK(param_dicts) or S_ERROR """ # API initialization is required to get an up-to-date configuration from the CS csAPI = CSAPI() csAPI.initialize() try: mqService, mqType, mqName = mqURI.split("::") except ValueError: return S_ERROR("Bad format of mqURI address:%s" % (mqURI)) result = gConfig.getConfigurationTree("/Resources/MQServices", mqService, mqType, mqName) if not result["OK"] or not result["Value"]: return S_ERROR( "Requested destination not found in the CS: %s::%s::%s" % (mqService, mqType, mqName)) mqDestinationPath = None for path, value in result["Value"].items(): if not value and path.endswith(mqName): mqDestinationPath = path # set-up internal parameter depending on the destination type tmp = mqDestinationPath.split("Queues")[0].split("Topics") servicePath = tmp[0] serviceDict = {} if len(tmp) > 1: serviceDict["Topic"] = mqName else: serviceDict["Queue"] = mqName result = gConfig.getOptionsDict(servicePath) if not result["OK"]: return result serviceDict.update(result["Value"]) result = gConfig.getOptionsDict(mqDestinationPath) if not result["OK"]: return result serviceDict.update(result["Value"]) return S_OK(serviceDict)
def __init__(self, cliParams): from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI self.modifiedCS = False self.softSec = "/Operations/Defaults/AvailableTarBalls" self.mailadress = '*****@*****.**' self.cliParams = cliParams self.parameter = dict(softSec=self.softSec, platform=cliParams.platform, version=cliParams.version, basepath=cliParams.basePath, initsctipt=cliParams.initScriptLocation) self.applications = cliParams.applicationSet self.detmodels = {} self.csAPI = CSAPI()
def initialize(self): """Gets run paramaters from the configuration""" self.addressTo = self.am_getOption("MailTo", self.addressTo) self.addressFrom = self.am_getOption("MailFrom", self.addressFrom) # Create a list of alternative bdii urls self.alternativeBDIIs = self.am_getOption("AlternativeBDIIs", self.alternativeBDIIs) self.host = self.am_getOption("Host", self.host) self.injectSingleCoreQueues = self.am_getOption( "InjectSingleCoreQueues", self.injectSingleCoreQueues) # Check if the bdii url is appended by a port number, if not append the default 2170 for index, url in enumerate(self.alternativeBDIIs): if not url.split(":")[-1].isdigit(): self.alternativeBDIIs[index] += ":2170" if self.addressTo and self.addressFrom: self.log.info("MailTo", self.addressTo) self.log.info("MailFrom", self.addressFrom) if self.alternativeBDIIs: self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs) self.processCEs = self.am_getOption("ProcessCEs", self.processCEs) self.selectedSites = self.am_getOption("SelectedSites", []) self.dryRun = self.am_getOption("DryRun", self.dryRun) self.voName = self.am_getOption("VirtualOrganization", self.voName) if not self.voName: self.voName = self.am_getOption("VO", []) if not self.voName or (len(self.voName) == 1 and self.voName[0].lower() == "all"): # Get all VOs defined in the configuration self.voName = [] result = getVOs() if result["OK"]: vos = result["Value"] for vo in vos: vomsVO = getVOOption(vo, "VOMSName") if vomsVO: self.voName.append(vomsVO) if self.voName: self.log.info("Agent will manage VO(s) %s" % self.voName) else: self.log.fatal("VirtualOrganization option not defined for agent") return S_ERROR() self.csAPI = CSAPI() return self.csAPI.initialize()
def __init__(self): """Internal initialization of the DIRAC Admin API.""" super(DiracAdmin, self).__init__() self.csAPI = CSAPI() self.dbg = False if gConfig.getValue(self.section + "/LogLevel", "DEBUG") == "DEBUG": self.dbg = True self.scratchDir = gConfig.getValue(self.section + "/ScratchDir", "/tmp") self.currentDir = os.getcwd() self.rssFlag = ResourceStatus().rssFlag self.sitestatus = SiteStatus()
def initialize( self ): # TODO: Have no default and if no mail is found then use the diracAdmin group # and resolve all associated mail addresses. self.addressTo = self.am_getOption( 'MailTo', self.addressTo ) self.addressFrom = self.am_getOption( 'MailFrom', self.addressFrom ) # create a list of alternative bdii urls self.alternativeBDIIs = self.am_getOption( 'AlternativeBDIIs', [] ) # check if the bdii url is appended by a port number, if not append the default 2170 for index, url in enumerate( self.alternativeBDIIs ): if not url.split( ':' )[-1].isdigit(): self.alternativeBDIIs[index] += ':2170' if self.addressTo and self.addressFrom: self.log.info( "MailTo", self.addressTo ) self.log.info( "MailFrom", self.addressFrom ) if self.alternativeBDIIs : self.log.info( "AlternativeBDII URLs:", self.alternativeBDIIs ) self.subject = "CE2CSAgent" # This sets the Default Proxy to used as that defined under # /Operations/Shifter/TestManager # the shifterProxy option in the Configuration can be used to change this default. self.am_setOption( 'shifterProxy', 'TestManager' ) self.voName = self.am_getOption( 'VirtualOrganization', self.voName ) if not self.voName: self.voName = getVO() if not self.voName: self.log.fatal( "VO option not defined for agent" ) return S_ERROR() self.csAPI = CSAPI() return self.csAPI.initialize()
def beginExecution(self): """Reload the configurations before every cycle.""" self.setup = self.am_getOption("Setup", self.setup) self.enabled = self.am_getOption("EnableFlag", self.enabled) self.restartAgents = self.am_getOption("RestartAgents", self.restartAgents) self.restartExecutors = self.am_getOption("RestartExecutors", self.restartExecutors) self.restartServices = self.am_getOption("RestartServices", self.restartServices) self.diracLocation = os.environ.get("DIRAC", self.diracLocation) self.addressTo = self.am_getOption('MailTo', self.addressTo) self.addressFrom = self.am_getOption('MailFrom', self.addressFrom) self.controlComponents = self.am_getOption('ControlComponents', self.controlComponents) self.commitURLs = self.am_getOption('CommitURLs', self.commitURLs) self.csAPI = CSAPI() res = self.getRunningInstances(instanceType='Agents') if not res["OK"]: return S_ERROR("Failure to get running agents") self.agents = res["Value"] res = self.getRunningInstances(instanceType='Executors') if not res["OK"]: return S_ERROR("Failure to get running executors") self.executors = res["Value"] res = self.getRunningInstances(instanceType='Services') if not res["OK"]: return S_ERROR("Failure to get running services") self.services = res["Value"] self.accounting.clear() return S_OK()
def initialize( self ): """ Standard constructor """ try: self.rsDB = ResourceStatusDB() self.rmDB = ResourceManagementDB() self.StorageElementToBeChecked = Queue.Queue() self.StorageElementInCheck = [] self.maxNumberOfThreads = self.am_getOption( 'maxThreadsInPool', 1 ) self.threadPool = ThreadPool( self.maxNumberOfThreads, self.maxNumberOfThreads ) if not self.threadPool: self.log.error( 'Can not create Thread Pool' ) return S_ERROR( 'Can not create Thread Pool' ) self.setup = getSetup()[ 'Value' ] self.VOExtension = getExt() self.StorageElsWriteFreqs = CheckingFreqs[ 'StorageElsWriteFreqs' ] self.nc = NotificationClient() self.diracAdmin = DiracAdmin() self.csAPI = CSAPI() for _i in xrange( self.maxNumberOfThreads ): self.threadPool.generateJobAndQueueIt( self._executeCheck, args = ( None, ) ) return S_OK() except Exception: errorStr = "StElWriteInspectorAgent initialization" gLogger.exception( errorStr ) return S_ERROR( errorStr )
def insertCSSection( self, path, pardict ): """ insert a section and values (or subsections) into the CS :param str path: full path of the new section :param str pardict: dictionary of key values in the new section, values can also be dictionaries :return: S_OK(), S_ERROR() """ from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI if self.csapi is None: self.csapi = CSAPI() for key, value in pardict.iteritems(): newSectionPath = os.path.join(path,key) gLogger.debug( "Adding to cs %s : %s " % ( newSectionPath, value ) ) self.csapi.createSection( path ) if isinstance( value, dict ): res = self.insertCSSection( newSectionPath, value ) else: res = self.csapi.setOption( newSectionPath, value ) if not res['OK']: return res else: gLogger.notice( "Added to CS: %s " % res['Value'] ) return S_OK("Added all things to cs")
def __init__(self): """Internal initialization of the DIRAC Admin API. """ super(DiracAdmin, self).__init__() self.csAPI = CSAPI() self.dbg = False if gConfig.getValue(self.section + '/LogLevel', 'DEBUG') == 'DEBUG': self.dbg = True self.scratchDir = gConfig.getValue(self.section + '/ScratchDir', '/tmp') self.currentDir = os.getcwd() self.rssFlag = ResourceStatus().rssFlag self.sitestatus = SiteStatus() self._siteSet = set(getSites().get('Value', []))
def initialize( self ): # client to connect to GOCDB self.GOCDBClient = GOCDBClient() # API needed to update configuration stored by CS self.csAPI = CSAPI() return self.csAPI.initialize()
def __init__( self, agentName, baseAgentName, properties ): AgentModule.__init__( self, agentName, baseAgentName, properties ) self.addressTo = '' self.addressFrom = '' self.voName = '' self.csAPI = CSAPI() self.subject = "CE2CSAgent" self.alternativeBDIIs = []
def __init__( self, vo, autoModifyUsers=True, autoAddUsers=True, autoDeleteUsers=False, autoLiftSuspendedStatus=False, syncPluginName=None, ): """VOMS2CSSynchronizer class constructor :param str vo: VO to be synced :param boolean autoModifyUsers: flag to automatically modify user data in CS :param autoAddUsers: flag to automatically add new users to CS :param autoDeleteUsers: flag to automatically delete users from CS if no more in VOMS :param autoLiftSuspendedStatus: flag to automatically remove Suspended status in CS :param syncPluginName: name of the plugin to validate or extend users' info :return: None """ self.log = gLogger.getSubLogger(self.__class__.__name__) self.csapi = CSAPI() self.vo = vo self.vomsVOName = getVOOption(vo, "VOMSName", "") if not self.vomsVOName: raise Exception("VOMS name not defined for VO %s" % vo) self.adminMsgs = {"Errors": [], "Info": []} self.vomsUserDict = {} self.autoModifyUsers = autoModifyUsers self.autoAddUsers = autoAddUsers self.autoDeleteUsers = autoDeleteUsers self.autoLiftSuspendedStatus = autoLiftSuspendedStatus self.voChanged = False self.syncPlugin = None if syncPluginName: objLoader = ObjectLoader() _class = objLoader.loadObject( "ConfigurationSystem.Client.SyncPlugins.%sSyncPlugin" % syncPluginName, "%sSyncPlugin" % syncPluginName ) if not _class["OK"]: raise Exception(_class["Message"]) self.syncPlugin = _class["Value"]()
def getMQParamsFromCS( mqURI ): """ Function gets parameters of a MQ destination (queue/topic) from the CS. Args: mqURI(str):Pseudo URI identifing the MQ service. It has the following format: mqConnection::DestinationType::DestinationName e.g. blabla.cern.ch::Queue::MyQueue1 mType(str): 'consumer' or 'producer' Returns: S_OK(param_dicts)/S_ERROR: """ # API initialization is required to get an up-to-date configuration from the CS csAPI = CSAPI() csAPI.initialize() try : mqService, mqType, mqName = mqURI.split( "::" ) except ValueError: return S_ERROR( 'Bad format of mqURI address:%s' % ( mqURI ) ) result = gConfig.getConfigurationTree( '/Resources/MQServices', mqService, mqType, mqName ) if not result['OK'] or len( result['Value'] ) == 0: return S_ERROR( 'Requested destination not found in the CS: %s::%s::%s' % ( mqService, mqType, mqName ) ) mqDestinationPath = None for path, value in result['Value'].iteritems(): if not value and path.endswith( mqName ): mqDestinationPath = path # set-up internal parameter depending on the destination type tmp = mqDestinationPath.split( 'Queue' )[0].split( 'Topic' ) servicePath = tmp[0] serviceDict = {} if len( tmp ) > 1: serviceDict['Topic'] = mqName else: serviceDict['Queue'] = mqName result = gConfig.getOptionsDict( servicePath ) if not result['OK']: return result serviceDict.update( result['Value'] ) result = gConfig.getOptionsDict( mqDestinationPath ) if not result['OK']: return result serviceDict.update( result['Value'] ) return S_OK( serviceDict )
def initialize(self): self.addressTo = self.am_getOption('MailTo', self.addressTo) self.addressFrom = self.am_getOption('MailFrom', self.addressFrom) # Create a list of alternative bdii urls self.alternativeBDIIs = self.am_getOption('AlternativeBDIIs', []) # Check if the bdii url is appended by a port number, if not append the default 2170 for index, url in enumerate(self.alternativeBDIIs): if not url.split(':')[-1].isdigit(): self.alternativeBDIIs[index] += ':2170' if self.addressTo and self.addressFrom: self.log.info("MailTo", self.addressTo) self.log.info("MailFrom", self.addressFrom) if self.alternativeBDIIs: self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs) self.subject = "Bdii2CSAgent" self.processCEs = self.am_getOption('ProcessCEs', True) self.processSEs = self.am_getOption('ProcessSEs', False) self.voName = self.am_getOption('VirtualOrganization', []) if not self.voName: self.voName = self.am_getOption('VO', []) if not self.voName or (len(self.voName) == 1 and self.voName[0].lower() == 'all'): # Get all VOs defined in the configuration self.voName = [] result = getVOs() if result['OK']: vos = result['Value'] for vo in vos: vomsVO = getVOOption(vo, "VOMSName") if vomsVO: self.voName.append(vomsVO) if self.voName: self.log.info("Agent will manage VO(s) %s" % self.voName) else: self.log.fatal("VirtualOrganization option not defined for agent") return S_ERROR() self.voBdiiCEDict = {} self.voBdiiSEDict = {} self.csAPI = CSAPI() return self.csAPI.initialize()
class TestComponentInstallation(unittest.TestCase): """ Contains methods for testing of separate elements """ def setUp(self): self.host = 'localhost' self.notificationPort = 9154 self.rootPwd = '' self.csClient = CSAPI() self.monitoringClient = ComponentMonitoringClient() self.client = SystemAdministratorClientCLI(self.host) self.csClient.downloadCSData() result = self.csClient.getCurrentCFG() if not result['OK']: raise Exception(result['Message']) cfg = result['Value'] setup = cfg.getOption('DIRAC/Setup', 'dirac-JenkinsSetup') self.frameworkSetup = cfg.getOption('DIRAC/Setups/' + setup + '/Framework') self.rootPwd = cfg.getOption('Systems/Databases/Password') self.diracPwd = self.rootPwd result = getProxyInfo() if not result['OK']: raise Exception(result['Message']) chain = result['Value']['chain'] result = chain.getCertInChain(-1) if not result['OK']: raise Exception(result['Message']) result = result['Value'].getSubjectDN() if not result['OK']: raise Exception(result['Message']) userDN = result['Value'] result = getUsernameForDN(userDN) if not result['OK']: raise Exception(result['Message']) self.user = result['Value'] if not self.user: self.user = '******' def tearDown(self): pass
class TestComponentInstallation(unittest.TestCase): """ Contains methods for testing of separate elements """ def setUp(self): self.host = "localhost" self.notificationPort = 9154 self.rootPwd = "" self.csClient = CSAPI() self.monitoringClient = ComponentMonitoringClient() self.client = SystemAdministratorClientCLI(self.host) self.csClient.downloadCSData() result = self.csClient.getCurrentCFG() if not result["OK"]: raise Exception(result["Message"]) cfg = result["Value"] setup = cfg.getOption("DIRAC/Setup", "dirac-JenkinsSetup") self.frameworkSetup = cfg.getOption("DIRAC/Setups/" + setup + "/Framework") self.rootPwd = cfg.getOption("Systems/Databases/Password") self.diracPwd = self.rootPwd result = getProxyInfo() if not result["OK"]: raise Exception(result["Message"]) chain = result["Value"]["chain"] result = chain.getCertInChain(-1) if not result["OK"]: raise Exception(result["Message"]) result = result["Value"].getSubjectDN() if not result["OK"]: raise Exception(result["Message"]) userDN = result["Value"] result = getUsernameForDN(userDN) if not result["OK"]: raise Exception(result["Message"]) self.user = result["Value"] if not self.user: self.user = "******" def tearDown(self): pass
def initialize(self): """ Run at the agent initialization (normally every 500 cycles) """ # client to connect to GOCDB self.GOCDBClient = GOCDBClient() self.dryRun = self.am_getOption('DryRun', self.dryRun) # API needed to update configuration stored by CS self.csAPI = CSAPI() return self.csAPI.initialize()
def setUp(self): self.host = 'localhost' self.notificationPort = 9154 self.rootPwd = '' self.csClient = CSAPI() self.monitoringClient = ComponentMonitoringClient() self.client = SystemAdministratorClientCLI(self.host) self.csClient.downloadCSData() result = self.csClient.getCurrentCFG() if not result['OK']: raise Exception(result['Message']) cfg = result['Value'] setup = cfg.getOption('DIRAC/Setup', 'JenkinsSetup') self.frameworkSetup = cfg.getOption('DIRAC/Setups/' + setup + '/Framework') self.rootPwd = cfg.getOption('Systems/Databases/Password') self.diracPwd = self.rootPwd
def __setCSStorageElementStatus( self, elementName, statusType, status ): """ Sets on the CS the StorageElements status """ statuses = self.rssConfig.getConfigStatusType( 'StorageElement' ) if not statusType in statuses: gLogger.error( "%s is not a valid statusType" % statusType ) return S_ERROR( "%s is not a valid statusType: %s" % ( statusType, statuses ) ) csAPI = CSAPI() cs_path = "/Resources/StorageElements" csAPI.setOption( "%s/%s/%s" % ( cs_path, elementName, statusType ), status ) res = csAPI.commitChanges() if not res[ 'OK' ]: gLogger.warn( 'CS: %s' % res[ 'Message' ] ) return res
def test_configurationAutoUpdate(value1, value2): """ Test if service refresh his configuration. It sent a random value to the CS and check if Service can return it. """ csapi = CSAPI() # SETTING FIRST VALUE csapi.modifyValue("/DIRAC/Configuration/TestUpdateValue", value1) csapi.commitChanges() # Wait for automatic refresh (+1 to be sure that request is done) time.sleep(gConfigurationData.getPropagationTime() + 1) RPCClient("Framework/User").getTestValue() assert RPCClient("Framework/User").getTestValue()["Value"] == value1 # SETTING SECOND VALUE csapi.modifyValue("/DIRAC/Configuration/TestUpdateValue", value2) csapi.commitChanges() time.sleep(gConfigurationData.getPropagationTime() + 1) assert RPCClient("Framework/User").getTestValue()["Value"] == value2
def __init__( self ): """Internal initialization of the DIRAC Admin API. """ super( DiracAdmin, self ).__init__() self.csAPI = CSAPI() self.dbg = False if gConfig.getValue( self.section + '/LogLevel', 'DEBUG' ) == 'DEBUG': self.dbg = True self.scratchDir = gConfig.getValue( self.section + '/ScratchDir', '/tmp' ) self.currentDir = os.getcwd()
def __init__(self): """Internal initialization of the DIRAC Admin API. """ super(DiracAdmin, self).__init__() self.csAPI = CSAPI() self.dbg = False if gConfig.getValue(self.section + "/LogLevel", "DEBUG") == "DEBUG": self.dbg = True self.scratchDir = gConfig.getValue(self.section + "/ScratchDir", "/tmp") self.currentDir = os.getcwd()
def __init__( self ): from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI self.detmodels = {} self.lcgeo_env="lcgeo_DIR" self.ddhep_env="DD4HEP" self.softSec = "/Operations/Defaults/AvailableTarBalls" self.version = '' self.platform = 'x86_64-slc5-gcc43-opt' self.comment = "" self.name = "ddsim" self.csapi = CSAPI() self.tarBallName = None self.md5sum = None
def beginExecution(self): """Reload the configurations before every cycle.""" self.setup = self.am_getOption("Setup", self.setup) self.enabled = self.am_getOption("EnableFlag", self.enabled) self.restartAgents = self.am_getOption("RestartAgents", self.restartAgents) self.restartExecutors = self.am_getOption("RestartExecutors", self.restartExecutors) self.restartServices = self.am_getOption("RestartServices", self.restartServices) self.addressTo = self.am_getOption("MailTo", self.addressTo) self.addressFrom = self.am_getOption("MailFrom", self.addressFrom) self.controlComponents = self.am_getOption("ControlComponents", self.controlComponents) self.commitURLs = self.am_getOption("CommitURLs", self.commitURLs) self.doNotRestartInstancePattern = self.am_getOption( "DoNotRestartInstancePattern", self.doNotRestartInstancePattern) self.csAPI = CSAPI() res = self.getRunningInstances(instanceType="Agents") if not res["OK"]: return S_ERROR("Failure to get running agents") self.agents = res["Value"] res = self.getRunningInstances(instanceType="Executors") if not res["OK"]: return S_ERROR("Failure to get running executors") self.executors = res["Value"] res = self.getRunningInstances(instanceType="Services") if not res["OK"]: return S_ERROR("Failure to get running services") self.services = res["Value"] self.accounting.clear() return S_OK()
def __init__(self, cliParams ): from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI self.modifiedCS = False self.softSec = "/Operations/Defaults/AvailableTarBalls" self.mailadress = '*****@*****.**' self.cliParams = cliParams self.parameter = dict( softSec = self.softSec, platform = cliParams.platform, version = cliParams.version, basepath = cliParams.basePath, initsctipt = cliParams.initScriptLocation ) self.applications = cliParams.applicationSet self.detmodels = {} self.csAPI = CSAPI()
def initialize(self): """ Standard constructor """ try: self.rsDB = ResourceStatusDB() self.rmDB = ResourceManagementDB() self.ResourcesToBeChecked = Queue.Queue() self.ResourceNamesInCheck = [] self.maxNumberOfThreads = self.am_getOption('maxThreadsInPool', 1) self.threadPool = ThreadPool(self.maxNumberOfThreads, self.maxNumberOfThreads) if not self.threadPool: self.log.error('Can not create Thread Pool') return S_ERROR('Can not create Thread Pool') self.setup = getSetup()['Value'] self.VOExtension = getExt() configModule = __import__( self.VOExtension + "DIRAC.ResourceStatusSystem.Policy.Configurations", globals(), locals(), ['*']) self.Resources_check_freq = copy.deepcopy( configModule.Resources_check_freq) self.nc = NotificationClient() self.diracAdmin = DiracAdmin() self.csAPI = CSAPI() for i in xrange(self.maxNumberOfThreads): self.threadPool.generateJobAndQueueIt(self._executeCheck, args=(None, )) return S_OK() except Exception: errorStr = "RSInspectorAgent initialization" gLogger.exception(errorStr) return S_ERROR(errorStr)
def setUp(self): self.dirac = Dirac() csAPI = CSAPI() self.lfn5 = os.path.join(DESTINATION_PATH, "test_file_10MB_v5.bin") self.dir5 = os.path.dirname(self.lfn5) # local file, for now: self.fname = os.path.basename(self.lfn5) random_dd(self.fname, 10) self.diracSE = "SE-1" try: self.fc = FileCatalogClient("DataManagement/MultiVOFileCatalog") except Exception: self.fail(" FileCatalog(['MultiVOFileCatalog']) raised Exception unexpectedly!\n" + traceback.format_exc()) return # add a replica self.fileadded = self.dirac.addFile(self.lfn5, self.fname, self.diracSE) self.assertTrue(self.fileadded["OK"])
def initialize( self ): self.addressTo = self.am_getOption( 'MailTo', self.addressTo ) self.addressFrom = self.am_getOption( 'MailFrom', self.addressFrom ) # Create a list of alternative bdii urls self.alternativeBDIIs = self.am_getOption( 'AlternativeBDIIs', [] ) # Check if the bdii url is appended by a port number, if not append the default 2170 for index, url in enumerate( self.alternativeBDIIs ): if not url.split( ':' )[-1].isdigit(): self.alternativeBDIIs[index] += ':2170' if self.addressTo and self.addressFrom: self.log.info( "MailTo", self.addressTo ) self.log.info( "MailFrom", self.addressFrom ) if self.alternativeBDIIs : self.log.info( "AlternativeBDII URLs:", self.alternativeBDIIs ) self.subject = "CE2CSAgent" self.processCEs = self.am_getOption( 'ProcessCEs', True ) self.processSEs = self.am_getOption( 'ProcessSEs', False ) self.voName = self.am_getOption( 'VirtualOrganization', [] ) if not self.voName: self.voName = self.am_getOption( 'VO', [] ) if not self.voName or ( len( self.voName ) == 1 and self.voName[0].lower() == 'all' ): # Get all VOs defined in the configuration self.voName = [] result = getVOs() if result['OK']: vos = result['Value'] for vo in vos: vomsVO = getVOOption( vo, "VOMSName" ) if vomsVO: self.voName.append( vomsVO ) if self.voName: self.log.info( "Agent will manage VO(s) %s" % self.voName ) else: self.log.fatal( "VirtualOrganization option not defined for agent" ) return S_ERROR() self.voBdiiCEDict = {} self.voBdiiSEDict = {} self.csAPI = CSAPI() return self.csAPI.initialize()
def setUp( self ): self.host = 'localhost' self.notificationPort = 9154 self.rootPwd = '' self.csClient = CSAPI() self.monitoringClient = ComponentMonitoringClient() self.client = SystemAdministratorClientCLI( self.host ) self.csClient.downloadCSData() result = self.csClient.getCurrentCFG() if not result[ 'OK' ]: raise Exception( result[ 'Message' ] ) cfg = result[ 'Value' ] setup = cfg.getOption( 'DIRAC/Setup', 'JenkinsSetup' ) self.frameworkSetup = cfg.getOption( 'DIRAC/Setups/' + setup + '/Framework' ) self.rootPwd = cfg.getOption( 'Systems/Databases/Password' ) self.diracPwd = self.rootPwd
def initialize( self ): """ Initialize the default parameters """ voNames = self.am_getOption( 'VO', [] ) if not voNames[0].lower() == "none": if voNames[0].lower() == "any": voNames = [] result = getVOMSVOs( voNames ) if not result['OK']: return result self.__voDict = result['Value'] self.log.notice( "VOs: %s" % self.__voDict.keys() ) self.csapi = CSAPI() self.dryRun = self.am_getOption( 'DryRun', self.dryRun ) self.autoAddUsers = self.am_getOption( 'AutoAddUsers', self.autoAddUsers ) self.autoModifyUsers = self.am_getOption( 'AutoModifyUsers', self.autoModifyUsers ) return S_OK()
def commit(self): """Commit the changes to the configuration system.""" # Perform all the appending operations at the end to only get from current config once. for path, value in chain(self._append_dict.iteritems(), self._append_unique_dict.iteritems()): section, option = path.rsplit('/', 1) self.add(section, option, value) self._num_changes += 1 result = CSAPI.commit(self) if not result['OK']: gLogger.error("Error while commit to CS", result['Message']) raise RuntimeError("Error while commit to CS") if self._num_changes: gLogger.notice("Successfully committed %d changes to CS\n" % self._num_changes) self._num_changes = 0 self._append_dict.clear() self._append_unique_dict.clear() else: gLogger.notice("No changes to commit")
def initialize( self ): self.__voDict = {} voNames = self.am_getOption( 'VO', [] ) if not voNames[0].lower() == "none": if voNames[0].lower() == "any": voNames = [] result = getVOMSVOs( voNames ) if not result['OK']: return result self.__voDict = result['Value'] self.__adminMsgs = {} self.csapi = CSAPI() self.voChanged = False self.log.notice( "VOs: %s" % self.__voDict.keys() ) self.autoAddUsers = self.am_getOption( 'AutoAddUsers', False ) self.autoModifyUsers = self.am_getOption( 'AutoModifyUsers', False ) self.autoSuspendUsers = self.am_getOption( 'AutoSuspendUsers', False ) return S_OK()
'Usage:', ' %s [option|cfgFile] ... DB ...' % Script.scriptName] ) ) Script.parseCommandLine() args = Script.getPositionalArgs() setupName = args[0] # Where to store outputs if not os.path.isdir( '%s/sandboxes' % setupName ): os.makedirs( '%s/sandboxes' % setupName ) # now updating the CS from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI csAPI = CSAPI() csAPI.setOption( 'Systems/WorkloadManagement/Production/Services/SandboxStore/BasePath', '%s/sandboxes' % setupName ) csAPI.setOption( 'Systems/WorkloadManagement/Production/Services/SandboxStore/LogLevel', 'DEBUG' ) # Now setting a SandboxSE as the following: # ProductionSandboxSE # { # BackendType = DISET # AccessProtocol.1 # { # Host = localhost # Port = 9196 # ProtocolName = DIP # Protocol = dips # Path = /scratch/workspace/%s/sandboxes % setupName
"\n".join( [ __doc__.split("\n")[1], "Usage:", " %s [option|cfgfile] ... DIRACSiteName GridSiteName CE [CE] ..." % Script.scriptName, "Arguments:", " DIRACSiteName: Name of the site for DIRAC in the form GRID.LOCATION.COUNTRY (ie:LCG.CERN.ch)", " GridSiteName: Name of the site in the Grid (ie: CERN-PROD)", " CE: Name of the CE to be included in the site (ie: ce111.cern.ch)", ] ) ) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() csAPI = CSAPI() if len(args) < 3: Script.showHelp() DIRACExit(-1) diracSiteName = args[0] gridSiteName = args[1] ces = args[2:] try: diracGridType, place, country = diracSiteName.split(".") except ValueError: gLogger.error("The DIRACSiteName should be of the form GRID.LOCATION.COUNTRY for example LCG.CERN.ch") DIRACExit(-1) result = getDIRACSiteName(gridSiteName)
class Bdii2CSAgent( AgentModule ): addressTo = '' addressFrom = '' voName = '' subject = "CE2CSAgent" alternativeBDIIs = [] def initialize( self ): self.addressTo = self.am_getOption( 'MailTo', self.addressTo ) self.addressFrom = self.am_getOption( 'MailFrom', self.addressFrom ) # Create a list of alternative bdii urls self.alternativeBDIIs = self.am_getOption( 'AlternativeBDIIs', [] ) # Check if the bdii url is appended by a port number, if not append the default 2170 for index, url in enumerate( self.alternativeBDIIs ): if not url.split( ':' )[-1].isdigit(): self.alternativeBDIIs[index] += ':2170' if self.addressTo and self.addressFrom: self.log.info( "MailTo", self.addressTo ) self.log.info( "MailFrom", self.addressFrom ) if self.alternativeBDIIs : self.log.info( "AlternativeBDII URLs:", self.alternativeBDIIs ) self.subject = "CE2CSAgent" self.processCEs = self.am_getOption( 'ProcessCEs', True ) self.processSEs = self.am_getOption( 'ProcessSEs', False ) self.voName = self.am_getOption( 'VirtualOrganization', [] ) if not self.voName: self.voName = self.am_getOption( 'VO', [] ) if not self.voName or ( len( self.voName ) == 1 and self.voName[0].lower() == 'all' ): # Get all VOs defined in the configuration self.voName = [] result = getVOs() if result['OK']: vos = result['Value'] for vo in vos: vomsVO = getVOOption( vo, "VOMSName" ) if vomsVO: self.voName.append( vomsVO ) if self.voName: self.log.info( "Agent will manage VO(s) %s" % self.voName ) else: self.log.fatal( "VirtualOrganization option not defined for agent" ) return S_ERROR() self.voBdiiCEDict = {} self.voBdiiSEDict = {} self.csAPI = CSAPI() return self.csAPI.initialize() def execute( self ): """ General agent execution method """ # Get a "fresh" copy of the CS data result = self.csAPI.downloadCSData() if not result['OK']: self.log.warn( "Could not download a fresh copy of the CS data", result[ 'Message' ] ) if self.processCEs: self.__lookForNewCEs() self.__updateCEs() if self.processSEs: self.__lookForNewSEs() self.__updateSEs() return S_OK() def __lookForNewCEs( self ): """ Look up BDII for CEs not yet present in the DIRAC CS """ bannedCEs = self.am_getOption( 'BannedCEs', [] ) result = getCEsFromCS() if not result['OK']: return result knownCEs = set( result['Value'] ) knownCEs = knownCEs.union( set( bannedCEs ) ) for vo in self.voName: result = self.__getBdiiCEInfo( vo ) if not result['OK']: continue bdiiInfo = result['Value'] result = getGridCEs( vo, bdiiInfo = bdiiInfo, ceBlackList = knownCEs ) if not result['OK']: self.log.error( 'Failed to get unused CEs', result['Message'] ) siteDict = result['Value'] body = '' for site in siteDict: newCEs = set( siteDict[site].keys() ) if not newCEs: continue ceString = '' for ce in newCEs: queueString = '' ceInfo = bdiiInfo[site]['CEs'][ce] ceString = "CE: %s, GOCDB Site Name: %s" % ( ce, site ) systemTuple = siteDict[site][ce]['System'] osString = "%s_%s_%s" % ( systemTuple ) newCEString = "\n%s\n%s\n" % ( ceString, osString ) for queue in ceInfo['Queues']: queueStatus = ceInfo['Queues'][queue].get( 'GlueCEStateStatus', 'UnknownStatus' ) if 'production' in queueStatus.lower(): ceType = ceInfo['Queues'][queue].get( 'GlueCEImplementationName', '' ) queueString += " %s %s %s\n" % ( queue, queueStatus, ceType ) if queueString: ceString = newCEString ceString += "Queues:\n" ceString += queueString if ceString: body += ceString if body: body = "\nWe are glad to inform You about new CE(s) possibly suitable for %s:\n" % vo + body body += "\n\nTo suppress information about CE add its name to BannedCEs list.\n" body += "Add new Sites/CEs for vo %s with the command:\n" % vo body += "dirac-admin-add-resources --vo %s --ce\n" % vo self.log.info( body ) if self.addressTo and self.addressFrom: notification = NotificationClient() result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False ) if not result['OK']: self.log.error( 'Can not send new site notification mail', result['Message'] ) return S_OK() def __getBdiiCEInfo( self, vo ): if vo in self.voBdiiCEDict: return S_OK( self.voBdiiCEDict[vo] ) self.log.info( "Check for available CEs for VO", vo ) result = getBdiiCEInfo( vo ) message = '' if not result['OK']: message = result['Message'] for bdii in self.alternativeBDIIs : result = getBdiiCEInfo( vo, host = bdii ) if result['OK']: break if not result['OK']: if message: self.log.error( "Error during BDII request", message ) else: self.log.error( "Error during BDII request", result['Message'] ) else: self.voBdiiCEDict[vo] = result['Value'] return result def __getBdiiSEInfo( self, vo ): if vo in self.voBdiiSEDict: return S_OK( self.voBdiiSEDict[vo] ) self.log.info( "Check for available SEs for VO", vo ) result = getBdiiSEInfo( vo ) message = '' if not result['OK']: message = result['Message'] for bdii in self.alternativeBDIIs : result = getBdiiSEInfo( vo, host = bdii ) if result['OK']: break if not result['OK']: if message: self.log.error( "Error during BDII request", message ) else: self.log.error( "Error during BDII request", result['Message'] ) else: self.voBdiiSEDict[vo] = result['Value'] return result def __updateCEs( self ): """ Update the Site/CE/queue settings in the CS if they were changed in the BDII """ bdiiChangeSet = set() for vo in self.voName: result = self.__getBdiiCEInfo( vo ) if not result['OK']: continue ceBdiiDict = result['Value'] result = getSiteUpdates( vo, bdiiInfo = ceBdiiDict, log = self.log ) if not result['OK']: continue bdiiChangeSet = bdiiChangeSet.union( result['Value'] ) # We have collected all the changes, consolidate VO settings result = self.__updateCS( bdiiChangeSet ) return result def __updateCS( self, bdiiChangeSet ): queueVODict = {} changeSet = set() for entry in bdiiChangeSet: section, option , _value, new_value = entry if option == "VO": queueVODict.setdefault( section, set() ) queueVODict[section] = queueVODict[section].union( set( new_value.split( ',' ) ) ) else: changeSet.add( entry ) for section, VOs in queueVODict.items(): changeSet.add( ( section, 'VO', '', ','.join( VOs ) ) ) if changeSet: changeList = list( changeSet ) changeList.sort() body = '\n'.join( [ "%s/%s %s -> %s" % entry for entry in changeList ] ) if body and self.addressTo and self.addressFrom: notification = NotificationClient() result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False ) if body: self.log.info( 'The following configuration changes were detected:' ) self.log.info( body ) for section, option, value, new_value in changeSet: if value == 'Unknown' or not value: self.csAPI.setOption( cfgPath( section, option ), new_value ) else: self.csAPI.modifyValue( cfgPath( section, option ), new_value ) result = self.csAPI.commit() if not result['OK']: self.log.error( "Error while committing to CS", result['Message'] ) else: self.log.info( "Successfully committed %d changes to CS" % len( changeList ) ) return result else: self.log.info( "No changes found" ) return S_OK() def __lookForNewSEs( self ): """ Look up BDII for SEs not yet present in the DIRAC CS """ bannedSEs = self.am_getOption( 'BannedSEs', [] ) result = getSEsFromCS() if not result['OK']: return result knownSEs = set( result['Value'] ) knownSEs = knownSEs.union( set( bannedSEs ) ) for vo in self.voName: result = self.__getBdiiSEInfo( vo ) if not result['OK']: continue bdiiInfo = result['Value'] result = getGridSRMs( vo, bdiiInfo = bdiiInfo, srmBlackList = knownSEs ) if not result['OK']: continue siteDict = result['Value'] body = '' for site in siteDict: newSEs = set( siteDict[site].keys() ) if not newSEs: continue for se in newSEs: body += '\n New SE %s available at site %s:\n' % ( se, site ) backend = siteDict[site][se]['SE'].get( 'GlueSEImplementationName', 'Unknown' ) size = siteDict[site][se]['SE'].get( 'GlueSESizeTotal', 'Unknown' ) body += ' Backend %s, Size %s' % ( backend, size ) if body: body = "\nWe are glad to inform You about new SE(s) possibly suitable for %s:\n" % vo + body body += "\n\nTo suppress information about an SE add its name to BannedSEs list.\n" body += "Add new SEs for vo %s with the command:\n" % vo body += "dirac-admin-add-resources --vo %s --se\n" % vo self.log.info( body ) if self.addressTo and self.addressFrom: notification = NotificationClient() result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False ) if not result['OK']: self.log.error( 'Can not send new site notification mail', result['Message'] ) return S_OK() def __updateSEs( self ): """ Update the Storage Element settings in the CS if they were changed in the BDII """ bdiiChangeSet = set() for vo in self.voName: result = self.__getBdiiSEInfo( vo ) if not result['OK']: continue seBdiiDict = result['Value'] result = getSRMUpdates( vo, bdiiInfo = seBdiiDict ) if not result['OK']: continue bdiiChangeSet = bdiiChangeSet.union( result['Value'] ) # We have collected all the changes, consolidate VO settings result = self.__updateCS( bdiiChangeSet ) return result
class CE2CSAgent( AgentModule ): addressTo = '' addressFrom = '' voName = '' subject = "CE2CSAgent" alternativeBDIIs = [] def initialize( self ): # TODO: Have no default and if no mail is found then use the diracAdmin group # and resolve all associated mail addresses. self.addressTo = self.am_getOption( 'MailTo', self.addressTo ) self.addressFrom = self.am_getOption( 'MailFrom', self.addressFrom ) # create a list of alternative bdii urls self.alternativeBDIIs = self.am_getOption( 'AlternativeBDIIs', [] ) # check if the bdii url is appended by a port number, if not append the default 2170 for index, url in enumerate( self.alternativeBDIIs ): if not url.split( ':' )[-1].isdigit(): self.alternativeBDIIs[index] += ':2170' if self.addressTo and self.addressFrom: self.log.info( "MailTo", self.addressTo ) self.log.info( "MailFrom", self.addressFrom ) if self.alternativeBDIIs : self.log.info( "AlternativeBDII URLs:", self.alternativeBDIIs ) self.subject = "CE2CSAgent" # This sets the Default Proxy to used as that defined under # /Operations/Shifter/TestManager # the shifterProxy option in the Configuration can be used to change this default. self.am_setOption( 'shifterProxy', 'TestManager' ) self.voName = self.am_getOption( 'VirtualOrganization', self.voName ) if not self.voName: self.voName = getVO() if not self.voName: self.log.fatal( "VO option not defined for agent" ) return S_ERROR() self.csAPI = CSAPI() return self.csAPI.initialize() def execute( self ): self.log.info( "Start Execution" ) result = getProxyInfo() if not result[ 'OK' ]: return result infoDict = result[ 'Value' ] self.log.info( formatProxyInfoAsString( infoDict ) ) #Get a "fresh" copy of the CS data result = self.csAPI.downloadCSData() if not result[ 'OK' ]: self.log.warn( "Could not download a fresh copy of the CS data", result[ 'Message' ] ) self.__lookForCE() self.__infoFromCE() self.log.info( "End Execution" ) return S_OK() def __checkAlternativeBDIISite( self, fun, *args ): if self.alternativeBDIIs: self.log.warn( "Trying to use alternative bdii sites" ) for site in self.alternativeBDIIs : self.log.info( "Trying to contact alternative bdii ", site ) if len( args ) == 1 : result = fun( args[0], host = site ) elif len( args ) == 2 : result = fun( args[0], vo = args[1], host = site ) if not result['OK'] : self.log.error ( "Problem contacting alternative bddii", result['Message'] ) elif result['OK'] : return result self.log.warn( "Also checking alternative BDII sites failed" ) return result def __lookForCE( self ): knownces = self.am_getOption( 'BannedCEs', [] ) result = gConfig.getSections( '/Resources/Sites' ) if not result['OK']: return grids = result['Value'] for grid in grids: result = gConfig.getSections( '/Resources/Sites/%s' % grid ) if not result['OK']: return sites = result['Value'] for site in sites: opt = gConfig.getOptionsDict( '/Resources/Sites/%s/%s' % ( grid, site ) )['Value'] ces = List.fromChar( opt.get( 'CE', '' ) ) knownces += ces response = ldapCEState( '', vo = self.voName ) if not response['OK']: self.log.error( "Error during BDII request", response['Message'] ) response = self.__checkAlternativeBDIISite( ldapCEState, '', self.voName ) return response newces = {} for queue in response['Value']: try: queuename = queue['GlueCEUniqueID'] except: continue cename = queuename.split( ":" )[0] if not cename in knownces: newces[cename] = None self.log.debug( "newce", cename ) body = "" possibleNewSites = [] for ce in newces.iterkeys(): response = ldapCluster( ce ) if not response['OK']: self.log.warn( "Error during BDII request", response['Message'] ) response = self.__checkAlternativeBDIISite( ldapCluster, ce ) continue clusters = response['Value'] if len( clusters ) != 1: self.log.warn( "Error in cluster length", " CE %s Length %d" % ( ce, len( clusters ) ) ) if len( clusters ) == 0: continue cluster = clusters[0] fkey = cluster.get( 'GlueForeignKey', [] ) if type( fkey ) == type( '' ): fkey = [fkey] nameBDII = None for entry in fkey: if entry.count( 'GlueSiteUniqueID' ): nameBDII = entry.split( '=' )[1] break if not nameBDII: continue cestring = "CE: %s, GOCDB Name: %s" % ( ce, nameBDII ) self.log.info( cestring ) response = ldapCE( ce ) if not response['OK']: self.log.warn( "Error during BDII request", response['Message'] ) response = self.__checkAlternativeBDIISite( ldapCE, ce ) continue ceinfos = response['Value'] if len( ceinfos ): ceinfo = ceinfos[0] systemName = ceinfo.get( 'GlueHostOperatingSystemName', 'Unknown' ) systemVersion = ceinfo.get( 'GlueHostOperatingSystemVersion', 'Unknown' ) systemRelease = ceinfo.get( 'GlueHostOperatingSystemRelease', 'Unknown' ) else: systemName = "Unknown" systemVersion = "Unknown" systemRelease = "Unknown" osstring = "SystemName: %s, SystemVersion: %s, SystemRelease: %s" % ( systemName, systemVersion, systemRelease ) self.log.info( osstring ) response = ldapCEState( ce, vo = self.voName ) if not response['OK']: self.log.warn( "Error during BDII request", response['Message'] ) response = self.__checkAlternativeBDIISite( ldapCEState, ce, self.voName ) continue newcestring = "\n\n%s\n%s" % ( cestring, osstring ) usefull = False cestates = response['Value'] for cestate in cestates: queuename = cestate.get( 'GlueCEUniqueID', 'UnknownName' ) queuestatus = cestate.get( 'GlueCEStateStatus', 'UnknownStatus' ) queuestring = "%s %s" % ( queuename, queuestatus ) self.log.info( queuestring ) newcestring += "\n%s" % queuestring if queuestatus.count( 'Production' ): usefull = True if usefull: body += newcestring possibleNewSites.append( 'dirac-admin-add-site DIRACSiteName %s %s' % ( nameBDII, ce ) ) if body: body = "We are glad to inform You about new CE(s) possibly suitable for %s:\n" % self.voName + body body += "\n\nTo suppress information about CE add its name to BannedCEs list." for possibleNewSite in possibleNewSites: body = "%s\n%s" % ( body, possibleNewSite ) self.log.info( body ) if self.addressTo and self.addressFrom: notification = NotificationClient() result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False ) return S_OK() def __infoFromCE( self ): sitesSection = cfgPath( 'Resources', 'Sites' ) result = gConfig.getSections( sitesSection ) if not result['OK']: return grids = result['Value'] changed = False body = "" for grid in grids: gridSection = cfgPath( sitesSection, grid ) result = gConfig.getSections( gridSection ) if not result['OK']: return sites = result['Value'] for site in sites: siteSection = cfgPath( gridSection, site ) opt = gConfig.getOptionsDict( siteSection )['Value'] name = opt.get( 'Name', '' ) if name: coor = opt.get( 'Coordinates', 'Unknown' ) mail = opt.get( 'Mail', 'Unknown' ) result = ldapSite( name ) if not result['OK']: self.log.warn( "BDII site %s: %s" % ( name, result['Message'] ) ) result = self.__checkAlternativeBDIISite( ldapSite, name ) if result['OK']: bdiisites = result['Value'] if len( bdiisites ) == 0: self.log.warn( name, "Error in bdii: leng = 0" ) else: if not len( bdiisites ) == 1: self.log.warn( name, "Warning in bdii: leng = %d" % len( bdiisites ) ) bdiisite = bdiisites[0] try: longitude = bdiisite['GlueSiteLongitude'] latitude = bdiisite['GlueSiteLatitude'] newcoor = "%s:%s" % ( longitude, latitude ) except: self.log.warn( "Error in bdii coor" ) newcoor = "Unknown" try: newmail = bdiisite['GlueSiteSysAdminContact'].split( ":" )[-1].strip() except: self.log.warn( "Error in bdii mail" ) newmail = "Unknown" self.log.debug( "%s %s %s" % ( name, newcoor, newmail ) ) if newcoor != coor: self.log.info( "%s" % ( name ), "%s -> %s" % ( coor, newcoor ) ) if coor == 'Unknown': self.csAPI.setOption( cfgPath( siteSection, 'Coordinates' ), newcoor ) else: self.csAPI.modifyValue( cfgPath( siteSection, 'Coordinates' ), newcoor ) changed = True if newmail != mail: self.log.info( "%s" % ( name ), "%s -> %s" % ( mail, newmail ) ) if mail == 'Unknown': self.csAPI.setOption( cfgPath( siteSection, 'Mail' ), newmail ) else: self.csAPI.modifyValue( cfgPath( siteSection, 'Mail' ), newmail ) changed = True celist = List.fromChar( opt.get( 'CE', '' ) ) if not celist: self.log.warn( site, 'Empty site list' ) continue # result = gConfig.getSections( cfgPath( siteSection,'CEs' ) # if not result['OK']: # self.log.debug( "Section CEs:", result['Message'] ) for ce in celist: ceSection = cfgPath( siteSection, 'CEs', ce ) result = gConfig.getOptionsDict( ceSection ) if not result['OK']: self.log.debug( "Section CE", result['Message'] ) wnTmpDir = 'Unknown' arch = 'Unknown' os = 'Unknown' si00 = 'Unknown' pilot = 'Unknown' cetype = 'Unknown' else: ceopt = result['Value'] wnTmpDir = ceopt.get( 'wnTmpDir', 'Unknown' ) arch = ceopt.get( 'architecture', 'Unknown' ) os = ceopt.get( 'OS', 'Unknown' ) si00 = ceopt.get( 'SI00', 'Unknown' ) pilot = ceopt.get( 'Pilot', 'Unknown' ) cetype = ceopt.get( 'CEType', 'Unknown' ) result = ldapCE( ce ) if not result['OK']: self.log.warn( 'Error in bdii for %s' % ce, result['Message'] ) result = self.__checkAlternativeBDIISite( ldapCE, ce ) continue try: bdiice = result['Value'][0] except: self.log.warn( 'Error in bdii for %s' % ce, result ) bdiice = None if bdiice: try: newwnTmpDir = bdiice['GlueSubClusterWNTmpDir'] except: newwnTmpDir = 'Unknown' if wnTmpDir != newwnTmpDir and newwnTmpDir != 'Unknown': section = cfgPath( ceSection, 'wnTmpDir' ) self.log.info( section, " -> ".join( ( wnTmpDir, newwnTmpDir ) ) ) if wnTmpDir == 'Unknown': self.csAPI.setOption( section, newwnTmpDir ) else: self.csAPI.modifyValue( section, newwnTmpDir ) changed = True try: newarch = bdiice['GlueHostArchitecturePlatformType'] except: newarch = 'Unknown' if arch != newarch and newarch != 'Unknown': section = cfgPath( ceSection, 'architecture' ) self.log.info( section, " -> ".join( ( arch, newarch ) ) ) if arch == 'Unknown': self.csAPI.setOption( section, newarch ) else: self.csAPI.modifyValue( section, newarch ) changed = True try: newos = '_'.join( ( bdiice['GlueHostOperatingSystemName'], bdiice['GlueHostOperatingSystemVersion'], bdiice['GlueHostOperatingSystemRelease'] ) ) except: newos = 'Unknown' if os != newos and newos != 'Unknown': section = cfgPath( ceSection, 'OS' ) self.log.info( section, " -> ".join( ( os, newos ) ) ) if os == 'Unknown': self.csAPI.setOption( section, newos ) else: self.csAPI.modifyValue( section, newos ) changed = True body = body + "OS was changed %s -> %s for %s at %s\n" % ( os, newos, ce, site ) try: newsi00 = bdiice['GlueHostBenchmarkSI00'] except: newsi00 = 'Unknown' if si00 != newsi00 and newsi00 != 'Unknown': section = cfgPath( ceSection, 'SI00' ) self.log.info( section, " -> ".join( ( si00, newsi00 ) ) ) if si00 == 'Unknown': self.csAPI.setOption( section, newsi00 ) else: self.csAPI.modifyValue( section, newsi00 ) changed = True try: rte = bdiice['GlueHostApplicationSoftwareRunTimeEnvironment'] if self.voName.lower() == 'lhcb': if 'VO-lhcb-pilot' in rte: newpilot = 'True' else: newpilot = 'False' else: newpilot = 'Unknown' except: newpilot = 'Unknown' if pilot != newpilot and newpilot != 'Unknown': section = cfgPath( ceSection, 'Pilot' ) self.log.info( section, " -> ".join( ( pilot, newpilot ) ) ) if pilot == 'Unknown': self.csAPI.setOption( section, newpilot ) else: self.csAPI.modifyValue( section, newpilot ) changed = True result = ldapCEState( ce, vo = self.voName ) #getBDIICEVOView if not result['OK']: self.log.warn( 'Error in bdii for queue %s' % ce, result['Message'] ) result = self.__checkAlternativeBDIISite( ldapCEState, ce, self.voName ) continue try: queues = result['Value'] except: self.log.warn( 'Error in bdii for queue %s' % ce, result['Massage'] ) continue newcetype = 'Unknown' for queue in queues: try: queuetype = queue['GlueCEImplementationName'] except: queuetype = 'Unknown' if newcetype == 'Unknown': newcetype = queuetype else: if queuetype != newcetype: self.log.warn( 'Error in bdii for ce %s ' % ce, 'different cetypes %s %s' % ( newcetype, queuetype ) ) if newcetype=='ARC-CE': newcetype = 'ARC' if cetype != newcetype and newcetype != 'Unknown': section = cfgPath( ceSection, 'CEType' ) self.log.info( section, " -> ".join( ( cetype, newcetype ) ) ) if cetype == 'Unknown': self.csAPI.setOption( section, newcetype ) else: self.csAPI.modifyValue( section, newcetype ) changed = True for queue in queues: try: queueName = queue['GlueCEUniqueID'].split( '/' )[-1] except: self.log.warn( 'error in queuename ', queue ) continue try: newmaxCPUTime = queue['GlueCEPolicyMaxCPUTime'] except: newmaxCPUTime = None newsi00 = None try: caps = queue['GlueCECapability'] if type( caps ) == type( '' ): caps = [caps] for cap in caps: if cap.count( 'CPUScalingReferenceSI00' ): newsi00 = cap.split( '=' )[-1] except: newsi00 = None queueSection = cfgPath( ceSection, 'Queues', queueName ) result = gConfig.getOptionsDict( queueSection ) if not result['OK']: self.log.warn( "Section Queues", result['Message'] ) maxCPUTime = 'Unknown' si00 = 'Unknown' else: queueopt = result['Value'] maxCPUTime = queueopt.get( 'maxCPUTime', 'Unknown' ) si00 = queueopt.get( 'SI00', 'Unknown' ) if newmaxCPUTime and ( maxCPUTime != newmaxCPUTime ): section = cfgPath( queueSection, 'maxCPUTime' ) self.log.info( section, " -> ".join( ( maxCPUTime, newmaxCPUTime ) ) ) if maxCPUTime == 'Unknown': self.csAPI.setOption( section, newmaxCPUTime ) else: self.csAPI.modifyValue( section, newmaxCPUTime ) changed = True if newsi00 and ( si00 != newsi00 ): section = cfgPath( queueSection, 'SI00' ) self.log.info( section, " -> ".join( ( si00, newsi00 ) ) ) if si00 == 'Unknown': self.csAPI.setOption( section, newsi00 ) else: self.csAPI.modifyValue( section, newsi00 ) changed = True if changed: self.log.info( body ) if body and self.addressTo and self.addressFrom: notification = NotificationClient() result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False ) return self.csAPI.commit() else: self.log.info( "No changes found" ) return S_OK()
class DDSimTarMaker( object ): """ create a tarball of the DDSim release """ def __init__( self ): self.detmodels = {} self.lcgeo_env="lcgeo_DIR" self.ddhep_env="DD4HEP" self.softSec = "/Operations/Defaults/AvailableTarBalls" self.version = '' self.platform = 'x86_64-slc5-gcc43-opt' self.comment = "" self.name = "ddsim" self.csapi = None self.tarBallName = None self.md5sum = None def copyDetectorModels( self, basePath, folder, targetFolder ): """copy the compact folders to the targetFolder """ for root,dirs,_files in os.walk( os.path.join(basePath, folder) ): for direct in dirs: if root.endswith("compact"): ## the main xml file must have the same name as the folder, ILD and CLIC follow this convention already xmlPath = os.path.join( root, direct, direct+".xml") if os.path.exists( xmlPath ): self.detmodels[direct] = "detectors/"+direct+"/"+direct+".xml" copyFolder( os.path.join(root, direct), targetFolder ) def createTarBall( self, folder ): """create a tar ball from the folder tar zcf $TARBALLNAME $LIBFOLDER/* """ ##Create the Tarball if os.path.exists(self.tarBallName): os.remove(self.tarBallName) gLogger.notice("Creating Tarball...") myappTar = tarfile.open(self.tarBallName, "w:gz") myappTar.add(folder, self.tarBallName[:-4]) myappTar.close() self.md5sum = md5.md5(open( self.tarBallName, 'r' ).read()).hexdigest() gLogger.notice("...Done") return S_OK( "Created Tarball") def parseArgs( self ): """ parse the command line arguments""" if len(sys.argv) != 3: raise RuntimeError( "Wrong number of arguments in call: '%s'" % " ".join(sys.argv) ) self.name = sys.argv[1] self.version = sys.argv[2] self.tarBallName = "%s%s.tgz" % (self.name, self.version) def checkEnvironment( self ): """ check if dd4hep and lcgeo are in the environment """ for var in [ self.ddhep_env, self.lcgeo_env , 'ROOTSYS' ]: if var not in os.environ: raise RuntimeError( "%s is not set" % var ) return os.environ[self.ddhep_env], os.environ[self.lcgeo_env], os.environ['ROOTSYS'] def createCSEntry( self ): """add the entries for this version into the Configuration System <version> { TarBall = ddsim<version>.tgz AdditionalEnvVar { ROOTSYS = /cvmfs/ilc.desy.de/sw/x86_64_gcc44_sl6/root/5.34.30 G4INSTALL = /cvmfs/ilc.desy.de/sw/x86_64_gcc44_sl6/geant4/10.01 G4DATA = /cvmfs/ilc.desy.de/sw/x86_64_gcc44_sl6/geant4/10.01/share/Geant4-10.1.0/data } Overwrite = True } Operations/DDSimDetectorModels/<Version> { CLIC_o2_v03 = detectors/CLIC_o2_v03/CLIC_o2_v03.xml ... } """ #FIXME: Get root and geant4 location from environment, make sure it is cvmfs csParameter = { "TarBall": self.tarBallName, "AdditionalEnvVar": { "ROOTSYS" : os.environ.get("ROOTSYS"), "G4INSTALL" : os.environ.get("G4INSTALL"), "G4DATA" : os.environ.get("G4DATA"), }, "Md5Sum": self.md5sum, } pars = dict( platform=self.platform, name="ddsim", version=self.version ) csPath = os.path.join( self.softSec , "%(platform)s/%(name)s/%(version)s/" % pars ) pprint(csParameter) result = self.insertCSSection( csPath, csParameter ) csPathModels = "Operations/Defaults/DDSimDetectorModels" csModels = { self.version : self.detmodels } pprint(csModels) result = self.insertCSSection( csPathModels, csModels ) if self.csapi is not None: resProxy = checkOrGetGroupProxy( "diracAdmin" ) if not resProxy['OK']: gLogger.error( "Failed to get AdminProxy", resProxy['Message'] ) raise RuntimeError( "Failed to get diracAdminProxy" ) self.csapi.commit() if not result['OK']: gLogger.error( "Failed to create CS Section", result['Message'] ) raise RuntimeError( "Failed to create CS Section" ) def insertCSSection( self, path, pardict ): """ insert a section and values (or subsections) into the CS :param str path: full path of the new section :param str pardict: dictionary of key values in the new section, values can also be dictionaries :return: S_OK(), S_ERROR() """ from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI if self.csapi is None: self.csapi = CSAPI() for key, value in pardict.iteritems(): newSectionPath = os.path.join(path,key) gLogger.debug( "Adding to cs %s : %s " % ( newSectionPath, value ) ) self.csapi.createSection( path ) if isinstance( value, dict ): res = self.insertCSSection( newSectionPath, value ) else: res = self.csapi.setOption( newSectionPath, value ) if not res['OK']: return res else: gLogger.notice( "Added to CS: %s " % res['Value'] ) return S_OK("Added all things to cs") def createDDSimTarBall( self ): """ do everything to create the DDSim tarball""" self.parseArgs() ddBase, lcgeoBase, _rootsys = self.checkEnvironment() realTargetFolder = os.path.join( os.getcwd(), self.name+self.version ) targetFolder = os.path.join( os.getcwd(), "temp", self.name+self.version ) for folder in (targetFolder, targetFolder+"/lib"): try: os.makedirs( folder ) except OSError: pass libraries = set() rootmaps = set() dd4hepLibPath = getLibraryPath( ddBase ) lcgeoPath = getLibraryPath( lcgeoBase ) self.copyDetectorModels( lcgeoBase, "CLIC" , targetFolder+"/detectors" ) self.copyDetectorModels( lcgeoBase, "ILD" , targetFolder+"/detectors" ) copyFolder( ddBase+"/DDDetectors/compact", realTargetFolder.rstrip("/")+"/DDDetectors") libraries.update( getFiles( dd4hepLibPath, ".so") ) libraries.update( getFiles( lcgeoPath, ".so" ) ) rootmaps.update( getFiles( dd4hepLibPath, ".rootmap") ) rootmaps.update( getFiles( lcgeoPath, ".rootmap" ) ) pprint( libraries ) pprint( rootmaps ) allLibs = set() for lib in libraries: allLibs.update( getDependentLibraries(lib) ) ### remote root and geant4 libraries, we pick them up from allLibs = set( [ lib for lib in allLibs if not ( "/geant4/" in lib.lower() or "/root/" in lib.lower()) ] ) print allLibs copyLibraries( libraries, targetFolder+"/lib" ) copyLibraries( allLibs, targetFolder+"/lib" ) copyLibraries( rootmaps, targetFolder+"/lib" ) getPythonStuff( ddBase+"/python" , targetFolder+"/lib/") getPythonStuff( lcgeoBase+"/lib/python", targetFolder+"/lib/" ) getPythonStuff( lcgeoBase+"/bin/ddsim", targetFolder+"/bin/" ) ##Should get this from CVMFS #getRootStuff( rootsys, targetFolder+"/ROOT" ) copyFolder( targetFolder+"/", realTargetFolder.rstrip("/") ) killRPath( realTargetFolder ) resolveLinks( realTargetFolder+"/lib" ) removeSystemLibraries( realTargetFolder+"/lib" ) #removeSystemLibraries( realTargetFolder+"/ROOT/lib" ) print self.detmodels self.createTarBall( realTargetFolder ) self.createCSEntry()
def checkUnusedSEs(): global vo, dry result = getGridSRMs( vo, unUsed = True ) if not result['OK']: gLogger.error( 'Failed to look up SRMs in BDII', result['Message'] ) siteSRMDict = result['Value'] # Evaluate VOs result = getVOs() if result['OK']: csVOs = set( result['Value'] ) else: csVOs = set( [vo] ) changeSetFull = set() for site in siteSRMDict: for gridSE in siteSRMDict[site]: changeSet = set() seDict = siteSRMDict[site][gridSE]['SE'] srmDict = siteSRMDict[site][gridSE]['SRM'] # Check the SRM version version = srmDict.get( 'GlueServiceVersion', '' ) if not ( version and version.startswith( '2' ) ): gLogger.debug( 'Skipping SRM service with version %s' % version ) continue result = getDIRACSiteName( site ) if not result['OK']: gLogger.notice( 'Unused se %s is detected at unused site %s' % ( gridSE, site ) ) gLogger.notice( 'Consider adding site %s to the DIRAC CS' % site ) continue diracSites = result['Value'] yn = raw_input( '\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: ' % ( gridSE, str( diracSites ) ) ) if not yn or yn.lower().startswith( 'y' ): if len( diracSites ) > 1: prompt = 'Which DIRAC site the new SE should be attached to ?' for i, s in enumerate( diracSites ): prompt += '\n[%d] %s' % ( i, s ) prompt += '\nEnter your choice number: ' inp = raw_input( prompt ) try: ind = int( inp ) except: gLogger.notice( 'Can not interpret your choice: %s, try again later' % inp ) continue diracSite = diracSites[ind] else: diracSite = diracSites[0] domain, siteName, country = diracSite.split( '.' ) recName = '%s-disk' % siteName inp = raw_input( 'Give a DIRAC name to the grid SE %s, default %s : ' % ( gridSE, recName ) ) diracSEName = inp if not inp: diracSEName = recName gLogger.notice( 'Adding new SE %s at site %s' % ( diracSEName, diracSite ) ) seSection = cfgPath( '/Resources/StorageElements', diracSEName ) changeSet.add( ( seSection, 'BackendType', seDict.get( 'GlueSEImplementationName', 'Unknown' ) ) ) changeSet.add( ( seSection, 'Description', seDict.get( 'GlueSEName', 'Unknown' ) ) ) bdiiVOs = set( [ re.sub( '^VO:', '', rule ) for rule in srmDict.get( 'GlueServiceAccessControlBaseRule', [] ) ] ) seVOs = csVOs.intersection( bdiiVOs ) changeSet.add( ( seSection, 'VO', ','.join( seVOs ) ) ) accessSection = cfgPath( seSection, 'AccessProtocol.1' ) changeSet.add( ( accessSection, 'Protocol', 'srm' ) ) changeSet.add( ( accessSection, 'ProtocolName', 'SRM2' ) ) endPoint = srmDict.get( 'GlueServiceEndpoint', '' ) result = pfnparse( endPoint ) if not result['OK']: gLogger.error( 'Can not get the SRM service end point. Skipping ...' ) continue host = result['Value']['Host'] port = result['Value']['Port'] changeSet.add( ( accessSection, 'Host', host ) ) changeSet.add( ( accessSection, 'Port', port ) ) changeSet.add( ( accessSection, 'Access', 'remote' ) ) voPathSection = cfgPath( accessSection, 'VOPath' ) if 'VOPath' in seDict: path = seDict['VOPath'] voFromPath = os.path.basename( path ) if voFromPath != diracVO: gLogger.notice( '\n!!! Warning: non-conventional VO path: %s\n' % path ) changeSet.add( ( voPathSection, diracVO, path ) ) path = os.path.dirname( path ) else: # Try to guess the Path domain = '.'.join( host.split( '.' )[-2:] ) path = '/dpm/%s/home' % domain changeSet.add( ( accessSection, 'Path', path ) ) changeSet.add( ( accessSection, 'SpaceToken', '' ) ) changeSet.add( ( accessSection, 'WSUrl', '/srm/managerv2?SFN=' ) ) gLogger.notice( 'SE %s will be added with the following parameters' % diracSEName ) changeList = list( changeSet ) changeList.sort() for entry in changeList: gLogger.notice( entry ) yn = raw_input( 'Do you want to add new SE %s ? default yes [yes|no]: ' % diracSEName ) if not yn or yn.lower().startswith( 'y' ): changeSetFull = changeSetFull.union( changeSet ) if dry: if changeSetFull: gLogger.notice( 'Skipping commit of the new SE data in a dry run' ) else: gLogger.notice( "No new SE to be added" ) return S_OK() if changeSetFull: csAPI = CSAPI() csAPI.initialize() result = csAPI.downloadCSData() if not result['OK']: gLogger.error( 'Failed to initialize CSAPI object', result['Message'] ) DIRACExit( -1 ) changeList = list( changeSetFull ) changeList.sort() for section, option, value in changeList: csAPI.setOption( cfgPath( section, option ), value ) yn = raw_input( 'New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ' ) if not yn or yn.lower().startswith( 'y' ): result = csAPI.commit() if not result['OK']: gLogger.error( "Error while commit to CS", result['Message'] ) else: gLogger.notice( "Successfully committed %d changes to CS" % len( changeSetFull ) ) else: gLogger.notice( "No new SE to be added" ) return S_OK()