class TestComponentInstallation(unittest.TestCase): """ Contains methods for testing of separate elements """ def setUp(self): self.host = 'localhost' self.notificationPort = 9154 self.rootPwd = '' self.csClient = CSAPI() self.monitoringClient = ComponentMonitoringClient() self.client = SystemAdministratorClientCLI(self.host) self.csClient.downloadCSData() result = self.csClient.getCurrentCFG() if not result['OK']: raise Exception(result['Message']) cfg = result['Value'] setup = cfg.getOption('DIRAC/Setup', 'JenkinsSetup') self.frameworkSetup = cfg.getOption('DIRAC/Setups/' + setup + '/Framework') self.rootPwd = cfg.getOption('Systems/Databases/Password') self.diracPwd = self.rootPwd def tearDown(self): pass
class TestComponentInstallation( unittest.TestCase ): """ Contains methods for testing of separate elements """ def setUp( self ): self.host = 'localhost' self.notificationPort = 9154 self.rootPwd = '' self.csClient = CSAPI() self.monitoringClient = ComponentMonitoringClient() self.client = SystemAdministratorClientCLI( self.host ) self.csClient.downloadCSData() result = self.csClient.getCurrentCFG() if not result[ 'OK' ]: raise Exception( result[ 'Message' ] ) cfg = result[ 'Value' ] setup = cfg.getOption( 'DIRAC/Setup', 'JenkinsSetup' ) self.frameworkSetup = cfg.getOption( 'DIRAC/Setups/' + setup + '/Framework' ) self.rootPwd = cfg.getOption( 'Systems/Databases/Password' ) self.diracPwd = self.rootPwd def tearDown( self ): pass
class TestComponentInstallation(unittest.TestCase): """ Contains methods for testing of separate elements """ def setUp(self): self.host = 'localhost' self.notificationPort = 9154 self.rootPwd = '' self.csClient = CSAPI() self.monitoringClient = ComponentMonitoringClient() self.client = SystemAdministratorClientCLI(self.host) self.csClient.downloadCSData() result = self.csClient.getCurrentCFG() if not result['OK']: raise Exception(result['Message']) cfg = result['Value'] setup = cfg.getOption('DIRAC/Setup', 'dirac-JenkinsSetup') self.frameworkSetup = cfg.getOption('DIRAC/Setups/' + setup + '/Framework') self.rootPwd = cfg.getOption('Systems/Databases/Password') self.diracPwd = self.rootPwd result = getProxyInfo() if not result['OK']: raise Exception(result['Message']) chain = result['Value']['chain'] result = chain.getCertInChain(-1) if not result['OK']: raise Exception(result['Message']) result = result['Value'].getSubjectDN() if not result['OK']: raise Exception(result['Message']) userDN = result['Value'] result = getUsernameForDN(userDN) if not result['OK']: raise Exception(result['Message']) self.user = result['Value'] if not self.user: self.user = '******' def tearDown(self): pass
class TestComponentInstallation(unittest.TestCase): """ Contains methods for testing of separate elements """ def setUp(self): self.host = "localhost" self.notificationPort = 9154 self.rootPwd = "" self.csClient = CSAPI() self.monitoringClient = ComponentMonitoringClient() self.client = SystemAdministratorClientCLI(self.host) self.csClient.downloadCSData() result = self.csClient.getCurrentCFG() if not result["OK"]: raise Exception(result["Message"]) cfg = result["Value"] setup = cfg.getOption("DIRAC/Setup", "dirac-JenkinsSetup") self.frameworkSetup = cfg.getOption("DIRAC/Setups/" + setup + "/Framework") self.rootPwd = cfg.getOption("Systems/Databases/Password") self.diracPwd = self.rootPwd result = getProxyInfo() if not result["OK"]: raise Exception(result["Message"]) chain = result["Value"]["chain"] result = chain.getCertInChain(-1) if not result["OK"]: raise Exception(result["Message"]) result = result["Value"].getSubjectDN() if not result["OK"]: raise Exception(result["Message"]) userDN = result["Value"] result = getUsernameForDN(userDN) if not result["OK"]: raise Exception(result["Message"]) self.user = result["Value"] if not self.user: self.user = "******" def tearDown(self): pass
module = \ str( rDict[ compType ][ system ][ component ][ 'Module' ] ) record['Component']['System'] = system record['Component']['Module'] = module # Transform 'Services' into 'service', 'Agents' into 'agent' ... record['Component']['Type'] = compType.lower()[:-1] record['Host']['HostName'] = host record['Host']['CPU'] = cpu record['Installation']['Instance'] = component record['Installation'][ 'InstallationTime'] = datetime.utcnow() records.append(record) # Databases csClient = CSAPI() cfg = csClient.getCurrentCFG()['Value'] if hasMySQL: allDB = allDBResult['Value'] availableDB = availableDBResult['Value'] for db in allDB: # Check for DIRAC only databases if db in availableDB.keys() and db != 'InstalledComponentsDB': # Check for 'installed' databases isSection = cfg.isSection \ ( 'Systems/' + availableDB[ db ][ 'System' ] + '/' + cfg.getOption( 'DIRAC/Setups/' + setup + '/' + availableDB[ db ][ 'System' ] ) + '/Databases/' + db + '/' ) if isSection:
def main(): Script.registerSwitch( "C", "country", "Sort site names by country postfix (i.e. LCG.IHEP.cn, LCG.IN2P3.fr, LCG.IHEP.su)", sortBy) Script.registerSwitch("R", "reverse", "Reverse the sort order", isReverse) # Registering arguments will automatically add their description to the help menu Script.registerArgument([ "Section: Name of the subsection in '/Resources/Sites/' for sort (i.e. LCG DIRAC)" ], mandatory=False) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() result = getProxyInfo() if not result["OK"]: gLogger.error("Failed to get proxy information", result["Message"]) DIRACExit(2) proxy = result["Value"] if proxy["secondsLeft"] < 1: gLogger.error("Your proxy has expired, please create new one") DIRACExit(2) group = proxy["group"] if "CSAdministrator" not in getPropertiesForGroup(group): gLogger.error( "You must be CSAdministrator user to execute this script") gLogger.notice( "Please issue 'dirac-proxy-init -g [group with CSAdministrator Property]'" ) DIRACExit(2) cs = CSAPI() result = cs.getCurrentCFG() if not result["OK"]: gLogger.error("Failed to get copy of CS", result["Message"]) DIRACExit(2) cfg = result["Value"] if not cfg.isSection("Resources"): gLogger.error("Section '/Resources' is absent in CS") DIRACExit(2) if not cfg.isSection("Resources/Sites"): gLogger.error("Subsection '/Resources/Sites' is absent in CS") DIRACExit(2) if args and len(args) > 0: resultList = args[:] else: resultList = cfg["Resources"]["Sites"].listSections() hasRun = False isDirty = False for i in resultList: if not cfg.isSection("Resources/Sites/%s" % i): gLogger.error("Subsection /Resources/Sites/%s does not exists" % i) continue hasRun = True if SORTBYNAME: dirty = cfg["Resources"]["Sites"][i].sortAlphabetically( ascending=not REVERSE) else: dirty = cfg["Resources"]["Sites"][i].sortByKey(key=country, reverse=REVERSE) if dirty: isDirty = True if not hasRun: gLogger.notice( "Failed to find suitable subsections with site names to sort") DIRACExit(0) if not isDirty: gLogger.notice("Nothing to do, site names are already sorted") DIRACExit(0) timestamp = toString(dateTime()) stamp = "Site names are sorted by %s script at %s" % (Script.scriptName, timestamp) cs.setOptionComment("/Resources/Sites", stamp) result = cs.commit() if not result["OK"]: gLogger.error("Failed to commit changes to CS", result["Message"]) DIRACExit(2) gLogger.notice("Site names are sorted and committed to CS") DIRACExit(0)
def main(): global excludedHosts Script.registerSwitch( "e:", "exclude=", "Comma separated list of hosts to be excluded from the scanning process", setExcludedHosts) Script.parseCommandLine(ignoreErrors=False) componentType = "" # Get my setup mySetup = gConfig.getValue("DIRAC/Setup") # Retrieve information from all the hosts client = SystemAdministratorIntegrator(exclude=excludedHosts) resultAll = client.getOverallStatus() if not resultAll["OK"]: gLogger.error(resultAll["Message"]) DIRACexit(-1) # Retrieve user installing the component result = getProxyInfo() if result["OK"]: user = result["Value"]["username"] else: DIRACexit(-1) if not user: user = "******" for host in resultAll["Value"]: if not resultAll["Value"][host]["OK"]: # If the host cannot be contacted, exclude it and send message excludedHosts.append(host) result = NotificationClient().sendMail( Operations().getValue("EMail/Production", []), "Unreachable host", "\ndirac-populate-component-db: Could not fill the database with the components from unreachable host %s\n" % host, ) if not result["OK"]: gLogger.error( "Can not send unreachable host notification mail: %s" % result["Message"]) resultHosts = client.getHostInfo() if not resultHosts["OK"]: gLogger.error(resultHosts["Message"]) DIRACexit(-1) resultInfo = client.getInfo() if not resultInfo["OK"]: gLogger.error(resultInfo["Message"]) DIRACexit(-1) resultMySQL = client.getMySQLStatus() if not resultMySQL["OK"]: gLogger.error(resultMySQL["Message"]) DIRACexit(-1) resultAllDB = client.getDatabases() if not resultAllDB["OK"]: gLogger.error(resultAllDB["Message"]) DIRACexit(-1) resultAvailableDB = client.getAvailableDatabases() if not resultAvailableDB["OK"]: gLogger.error(resultAvailableDB["Message"]) DIRACexit(-1) records = [] finalSet = list(set(resultAll["Value"]) - set(excludedHosts)) for host in finalSet: hasMySQL = True result = resultAll["Value"][host] hostResult = resultHosts["Value"][host] infoResult = resultInfo["Value"][host] mySQLResult = resultMySQL["Value"][host] allDBResult = resultAllDB["Value"][host] availableDBResult = resultAvailableDB["Value"][host] if not result["OK"]: gLogger.error("Host %s: %s" % (host, result["Message"])) continue if not hostResult["OK"]: gLogger.error("Host %s: %s" % (host, hostResult["Message"])) continue if not infoResult["OK"]: gLogger.error("Host %s: %s" % (host, infoResult["Message"])) continue if mySQLResult["OK"]: if not allDBResult["OK"]: gLogger.error("Host %s: %s" % (host, allDBResult["Message"])) continue if not availableDBResult["OK"]: gLogger.error("Host %s: %s" % (host, availableDBResult["Message"])) continue else: hasMySQL = False setup = infoResult["Value"]["Setup"] if setup != mySetup: continue cpu = hostResult["Value"]["CPUModel"].strip() rDict = result["Value"] # Components other than databases for compType in rDict: if componentType and componentType != compType: continue for system in rDict[compType]: components = sorted(rDict[compType][system]) for component in components: record = {"Installation": {}, "Component": {}, "Host": {}} if rDict[compType][system][component][ "Installed"] and component != "ComponentMonitoring": runitStatus = str( rDict[compType][system][component]["RunitStatus"]) if runitStatus != "Unknown": module = str( rDict[compType][system][component]["Module"]) record["Component"]["System"] = system record["Component"]["Module"] = module # Transform 'Services' into 'service', 'Agents' into 'agent' ... record["Component"]["Type"] = compType.lower()[:-1] record["Host"]["HostName"] = host record["Host"]["CPU"] = cpu record["Installation"]["Instance"] = component record["Installation"][ "InstallationTime"] = datetime.utcnow() record["Installation"]["InstalledBy"] = user records.append(record) # Databases csClient = CSAPI() cfg = csClient.getCurrentCFG()["Value"] if hasMySQL: allDB = allDBResult["Value"] availableDB = availableDBResult["Value"] for db in allDB: # Check for DIRAC only databases if db in availableDB and db != "InstalledComponentsDB": # Check for 'installed' databases isSection = cfg.isSection( "Systems/" + availableDB[db]["System"] + "/" + cfg.getOption("DIRAC/Setups/" + setup + "/" + availableDB[db]["System"]) + "/Databases/" + db + "/") if isSection: record = { "Installation": {}, "Component": {}, "Host": {} } record["Component"]["System"] = availableDB[db][ "System"] record["Component"]["Module"] = db record["Component"]["Type"] = "DB" record["Host"]["HostName"] = host record["Host"]["CPU"] = cpu record["Installation"]["Instance"] = db record["Installation"][ "InstallationTime"] = datetime.utcnow() record["Installation"]["InstalledBy"] = user records.append(record) monitoringClient = ComponentMonitoringClient() # Add the installations to the database for record in records: result = MonitoringUtilities.monitorInstallation( record["Component"]["Type"], record["Component"]["System"], record["Installation"]["Instance"], record["Component"]["Module"], record["Host"]["CPU"], record["Host"]["HostName"], ) if not result["OK"]: gLogger.error(result["Message"])
for site in sites: result = getSiteName(site) siteName = result['Value'] gLogger.notice('Processing CondDB endpoint at site %s' % siteName) csapi.copySection( '/Resources/CondDB/%s' % site, '%s/Sites/%s/DBServer/CondDB' % (RESOURCES_NEW_SECTION, siteName)) csapi.setOptionComment( '%s/Sites/%s/DBServer' % (RESOURCES_NEW_SECTION, siteName), 'Database server resource') csapi.setOptionComment( '%s/Sites/%s/DBServer/CondDB' % (RESOURCES_NEW_SECTION, siteName), 'Conditions database') if __name__ == '__main__': if computingFlag: result = convertSites() if storageFlag: result = convertSEs() if catalogFlag: result = convertCatalogs() if transferFlag: result = convertTransfers() if dbFlag: result = convertDBServers() csapi.commitChanges() print csapi.getCurrentCFG()['Value'].serialize()
module = \ str( rDict[ compType ][ system ][ component ][ 'Module' ] ) record[ 'Component' ][ 'System' ] = system record[ 'Component' ][ 'Module' ] = module # Transform 'Services' into 'service', 'Agents' into 'agent' ... record[ 'Component' ][ 'Type' ] = compType.lower()[ :-1 ] record[ 'Host' ][ 'HostName' ] = host record[ 'Host' ][ 'CPU' ] = cpu record[ 'Installation' ][ 'Instance' ] = component record[ 'Installation' ][ 'InstallationTime' ] = datetime.utcnow() record[ 'Installation' ][ 'InstalledBy' ] = user records.append( record ) # Databases csClient = CSAPI() cfg = csClient.getCurrentCFG()[ 'Value' ] if hasMySQL: allDB = allDBResult[ 'Value' ] availableDB = availableDBResult[ 'Value' ] for db in allDB: # Check for DIRAC only databases if db in availableDB.keys() and db != 'InstalledComponentsDB': # Check for 'installed' databases isSection = cfg.isSection \ ( 'Systems/' + availableDB[ db ][ 'System' ] + '/' + cfg.getOption( 'DIRAC/Setups/' + setup + '/' + availableDB[ db ][ 'System' ] ) + '/Databases/' + db + '/' ) if isSection:
result = getProxyInfo() if not result["OK"]: gLogger.error("Failed to get proxy information", result["Message"]) DIRAC.exit(2) proxy = result["Value"] if proxy["secondsLeft"] < 1: gLogger.error("Your proxy has expired, please create new one") DIRAC.exit(2) group = proxy["group"] if not "CSAdministrator" in getPropertiesForGroup(group): gLogger.error("You must be CSAdministrator user to execute this script") gLogger.notice("Please issue 'dirac-proxy-init -g [group with CSAdministrator Property]'") DIRAC.exit(2) cs = CSAPI() result = cs.getCurrentCFG() if not result["OK"]: gLogger.error("Failed to get copy of CS", result["Message"]) DIRAC.exit(2) cfg = result["Value"] if not cfg.isSection("Resources"): gLogger.error("Section '/Resources' is absent in CS") DIRAC.exit(2) if not cfg.isSection("Resources/Sites"): gLogger.error("Subsection '/Resources/Sites' is absent in CS") DIRAC.exit(2) if args and len(args) > 0:
siteName = result['Value'] gLogger.notice( 'Processing CondDB endpoint at site %s' % siteName ) csapi.copySection( '/Resources/CondDB/%s' % site, '%s/Sites/%s/DBServer/CondDB' % (RESOURCES_NEW_SECTION,siteName) ) csapi.setOptionComment( '%s/Sites/%s/DBServer' % (RESOURCES_NEW_SECTION,siteName), 'Database server resource' ) csapi.setOptionComment( '%s/Sites/%s/DBServer/CondDB' % (RESOURCES_NEW_SECTION,siteName), 'Conditions database' ) if __name__ == '__main__': if computingFlag: result = convertSites() if storageFlag: result = convertSEs() if catalogFlag: result = convertCatalogs() if transferFlag: result = convertTransfers() if dbFlag: result = convertDBServers() csapi.commitChanges() print csapi.getCurrentCFG()['Value'].serialize()
gLogger.error("Failed to get proxy information", result["Message"]) DIRAC.exit(2) proxy = result["Value"] if proxy["secondsLeft"] < 1: gLogger.error("Your proxy has expired, please create new one") DIRAC.exit(2) group = proxy["group"] if not "CSAdministrator" in getPropertiesForGroup(group): gLogger.error("You must be CSAdministrator user to execute this script") gLogger.notice( "Please issue 'dirac-proxy-init -g [group with CSAdministrator Property]'" ) DIRAC.exit(2) cs = CSAPI() result = cs.getCurrentCFG() if not result["OK"]: gLogger.error("Failed to get copy of CS", result["Message"]) DIRAC.exit(2) cfg = result["Value"] if not cfg.isSection("Resources"): gLogger.error("Section '/Resources' is absent in CS") DIRAC.exit(2) if not cfg.isSection("Resources/Sites"): gLogger.error("Subsection '/Resources/Sites' is absent in CS") DIRAC.exit(2) if args and len(args) > 0: resultList = args[:]
def main(): global excludedHosts Script.registerSwitch( "e:", "exclude=", "Comma separated list of hosts to be excluded from the scanning process", setExcludedHosts) Script.parseCommandLine(ignoreErrors=False) componentType = '' # Get my setup mySetup = gConfig.getValue('DIRAC/Setup') # Retrieve information from all the hosts client = SystemAdministratorIntegrator(exclude=excludedHosts) resultAll = client.getOverallStatus() # Retrieve user installing the component result = getProxyInfo() if result['OK']: user = result['Value']['username'] else: DIRACexit(-1) if not user: user = '******' notificationClient = NotificationClient() for host in resultAll['Value']: if not resultAll['Value'][host]['OK']: # If the host cannot be contacted, exclude it and send message excludedHosts.append(host) result = notificationClient.sendMail( Operations().getValue('EMail/Production', []), 'Unreachable host', '\ndirac-populate-component-db: Could not fill the database with the components from unreachable host %s\n' % host) if not result['OK']: gLogger.error( 'Can not send unreachable host notification mail: %s' % result['Message']) if not resultAll['OK']: gLogger.error(resultAll['Message']) DIRACexit(-1) resultHosts = client.getHostInfo() if not resultHosts['OK']: gLogger.error(resultHosts['Message']) DIRACexit(-1) resultInfo = client.getInfo() if not resultInfo['OK']: gLogger.error(resultInfo['Message']) DIRACexit(-1) resultMySQL = client.getMySQLStatus() if not resultMySQL['OK']: gLogger.error(resultMySQL['Message']) DIRACexit(-1) resultAllDB = client.getDatabases() if not resultAllDB['OK']: gLogger.error(resultAllDB['Message']) DIRACexit(-1) resultAvailableDB = client.getAvailableDatabases() if not resultAvailableDB['OK']: gLogger.error(resultAvailableDB['Message']) DIRACexit(-1) records = [] finalSet = list(set(resultAll['Value']) - set(excludedHosts)) for host in finalSet: hasMySQL = True result = resultAll['Value'][host] hostResult = resultHosts['Value'][host] infoResult = resultInfo['Value'][host] mySQLResult = resultMySQL['Value'][host] allDBResult = resultAllDB['Value'][host] availableDBResult = resultAvailableDB['Value'][host] if not result['OK']: gLogger.error('Host %s: %s' % (host, result['Message'])) continue if not hostResult['OK']: gLogger.error('Host %s: %s' % (host, hostResult['Message'])) continue if not infoResult['OK']: gLogger.error('Host %s: %s' % (host, infoResult['Message'])) continue if mySQLResult['OK']: if not allDBResult['OK']: gLogger.error('Host %s: %s' % (host, allDBResult['Message'])) continue if not availableDBResult['OK']: gLogger.error('Host %s: %s' % (host, availableDBResult['Message'])) continue else: hasMySQL = False setup = infoResult['Value']['Setup'] if setup != mySetup: continue cpu = hostResult['Value']['CPUModel'].strip() rDict = result['Value'] # Components other than databases for compType in rDict: if componentType and componentType != compType: continue for system in rDict[compType]: components = sorted(rDict[compType][system]) for component in components: record = {'Installation': {}, 'Component': {}, 'Host': {}} if rDict[compType][system][component]['Installed'] and \ component != 'ComponentMonitoring': runitStatus = \ str(rDict[compType][system][component]['RunitStatus']) if runitStatus != 'Unknown': module = \ str(rDict[compType][system][component]['Module']) record['Component']['System'] = system record['Component']['Module'] = module # Transform 'Services' into 'service', 'Agents' into 'agent' ... record['Component']['Type'] = compType.lower()[:-1] record['Host']['HostName'] = host record['Host']['CPU'] = cpu record['Installation']['Instance'] = component record['Installation'][ 'InstallationTime'] = datetime.utcnow() record['Installation']['InstalledBy'] = user records.append(record) # Databases csClient = CSAPI() cfg = csClient.getCurrentCFG()['Value'] if hasMySQL: allDB = allDBResult['Value'] availableDB = availableDBResult['Value'] for db in allDB: # Check for DIRAC only databases if db in availableDB and db != 'InstalledComponentsDB': # Check for 'installed' databases isSection = cfg.isSection( 'Systems/' + availableDB[db]['System'] + '/' + cfg.getOption('DIRAC/Setups/' + setup + '/' + availableDB[db]['System']) + '/Databases/' + db + '/') if isSection: record = { 'Installation': {}, 'Component': {}, 'Host': {} } record['Component']['System'] = availableDB[db][ 'System'] record['Component']['Module'] = db record['Component']['Type'] = 'DB' record['Host']['HostName'] = host record['Host']['CPU'] = cpu record['Installation']['Instance'] = db record['Installation'][ 'InstallationTime'] = datetime.utcnow() record['Installation']['InstalledBy'] = user records.append(record) monitoringClient = ComponentMonitoringClient() # Add the installations to the database for record in records: result = MonitoringUtilities.monitorInstallation( record['Component']['Type'], record['Component']['System'], record['Installation']['Instance'], record['Component']['Module'], record['Host']['CPU'], record['Host']['HostName']) if not result['OK']: gLogger.error(result['Message'])