def loadConfig(db = "ProdAgentDB"): """ loadConfig method that loads the configuration of database base from ProdAgentConfig.xml and return it asdictionary object Argument: db : database configuration block to load Return: Dictionary containing the required configuration """ try: global first_access cfg = loadProdAgentConfiguration() dbConfig = cfg.getConfig(db) if (first_access == True): defaultConfig.update(dbConfig) first_access = False return dbConfig except StandardError, ex: msg = "%s.Config:" % db msg += "Unable to load ProdAgent Config for " + db msg += "%s\n" % ex logging.warning(msg)
def __init__(self): # TODO: The ResourceMonitor recreates this object in every call # Is there a way to reuse the DB interface? """ _TaskQueueThresholdMonitor_ This enhanced constructor creates an API to the TaskQueue (not to recreate it every time it is called). """ # Call our parent MonitorInterface.__init__(self) logging.debug("<<<<< Creating TaskQueueThresholdMonitor>>>>>") self.me = "TaskQueueThresholdMonitor" # Try to get the TaskQueue conf file from ProdAgentConfig.xml what = "TaskQueue" try: cfg = loadProdAgentConfiguration() self.tqConfig = cfg.getConfig(what) except StandardError, ex: msg = "%s.Config:" % what msg += "Unable to load ProdAgent Config for " + what msg += "%s\n" % ex logging.critical(msg) raise ProdAgentException(msg)
def getSiteInfoFromBase(sites, urls, ldaphost, dbp=None, samUrl=None): """ Collect all site info starting from LCG BDII Site names. Returns dictionary. """ use_phed=True try: if not dbp: from ProdAgentCore.Configuration import loadProdAgentConfiguration PAcfg = loadProdAgentConfiguration() PhedCfg = PAcfg['PhEDExConfig'] dbp = PhedCfg['DBPARAM'] DBParamFile,Section = dbp.split(':') if not os.path.isfile(DBParamFile): logging.error("Configured DBParamFile %s not found" % DBParamFile) use_phed=False except: print "Something went wrong with the Phedex config. Not using TMDB." use_phed=False #strip of text after "#", and remove site duplicates #bdiiNames = list(set([x['SiteName'].split("#")[0] for x in sites])) tmp_list = getBdii(sites, ldaphost) #tmp_list=getBdii(bdiiNames,ldaphost) #fcr info #ceuids_in_fcr=getFcr(urls) # for ceuid in tmp_list.keys(): # if ceuid in ceuids_in_fcr: # tmp_list[ceuid]['in_fcr']=True #tmp_list = [x for x in tmp_list if x['CEName'] in ceuids_in_fcr] tmp_list = getFcr(urls, tmp_list) #sam test info tmp_list = samCheck(tmp_list, samUrl) # if use_phed: # #TODO: Out of date, Do we want to track links here - # # esp if we are going to auto-create subscriptions # Phmap=getPhedexSEmap(DBParamFile,Section) # # for site in tmp_list: # #se_in_phedex=None # for se in site.values(): #['SEs'][:]: #tmp_list[ceuid]['SEs']: # se = se['SE'] # if not se in Phmap.values(): # ## this can happen more than once, # ## but should not cause problems # #se_in_phedex=se # site['SEs'].remove(se) # else: # print "None of the published SEs can be found in Phedex." # print " This is probably a bug somewhere." # print "Currnet SEs: %s, CEUID: %s" % ( # str(tmp_list[ceuid]['SEs']),ceuid) return tmp_list
def loadLSFConfig(): try: config = loadProdAgentConfiguration() except StandardError, ex : msg = "Error reading configuration: \n" msg += str(ex) logging.error(msg) raise RuntimeError, msg
def getResourceMonitorPlugin(): try: config = loadProdAgentConfiguration() compCfg = config.getConfig("ResourceMonitor") except StandardError, ex: msg = "Error reading configuration:\n" msg += str(ex) raise RuntimeError, msg
def alcaNewSelection(): if GlobalValues._alcaNewSelection == None: GlobalValues._alcaNewSelection = False paConfig = loadProdAgentConfiguration() runConfig = paConfig.getConfig( "RunConfig" ) parameter = "AlcaSubscription" if parameter in runConfig: GlobalValues._alcaSubName = runConfig[parameter] if GlobalValues._alcaDefSubName != GlobalValues._alcaSubName: GlobalValues._alcaNewSelection = True return GlobalValues._alcaNewSelection
def index(self,logFile) : config = loadProdAgentConfiguration() componentName = logFile.split("/")[0].strip() componentDir = config.getConfig(componentName)['ComponentDir'] componentDir = os.path.expandvars(componentDir) file = logFile.split("/")[1].strip() html = """<html><body><h2>%s for %s </h2>\n """ % (file, componentName) componentLog = open(componentDir+'/'+file).read().replace('\n','<br>') html += componentLog html += """</body></html>""" return html
def getGlobalDBS(self): """getGlobalDBS Gets Global DBS url form the ProdAgent Configuration """ try: config = loadProdAgentConfiguration() except StandardError, ex: msg = "Error reading configuration:\n" msg += str(ex) raise RuntimeError, msg
def getSuccessArchiveDir(): """ get the SuccessArchive dir from JobCleanup Component config """ try: config = loadProdAgentConfiguration() except StandardError, ex: msg = "Error: error reading configuration:\n" msg += str(ex) print msg sys.exit(1)
def getRequestInjectorConfig(): """ get the RequestInjector Component dir and the optional FirstRun """ try: config = loadProdAgentConfiguration() except StandardError, ex: msg = "Error: error reading configuration:\n" msg += str(ex) print msg sys.exit(1)
def getJobQueueConfig(): """ find out if we should use the jobQueue """ try: config = loadProdAgentConfiguration() except StandardError, ex: msg = "Error: error reading configuration:\n" msg += str(ex) print msg sys.exit(1)
def getDbox(self): from ProdAgentCore.Configuration import loadProdAgentConfiguration dbox = None try: config = loadProdAgentConfiguration() compCfg = config.getConfig("CrabServerConfigurations") dbox = compCfg["CacheDir"] except Exception, exc: logging.error(str(exc)) raise Exception("Problem loading server configuration info " + str(exc))
def __call__(self): """ _operator()_ Invoke the Bot to check each component """ cfgObject = loadProdAgentConfiguration() components = cfgObject.listComponents() restartedComponents = [] for component in components: logging.info("RestartBot: Checking %s" % component) compCfg = cfgObject.getConfig(component) compDir = compCfg['ComponentDir'] compDir = os.path.expandvars(compDir) daemonXml = os.path.join(compDir, "Daemon.xml") doRestart = False if not os.path.exists(daemonXml): doRestart = True daemon = DaemonDetails(daemonXml) if not daemon.isAlive(): doRestart = True if doRestart: restartedComponents.append(component) modRef = __import__(component, globals(), locals(), []) srcFile = inspect.getsourcefile(modRef) srcDir = os.path.dirname(srcFile) startup = os.path.join(srcDir, "Startup.py") if not os.path.exists(startup): msg = "Error starting component: %s\n" % component msg += "Startup file is not found:\n" msg += " %s \n" % startup logging.warning(msg) continue logging.info( "Starting Component %s:" % component) logging.info( "With: %s" % startup) os.system("%s %s " % (sys.executable, startup)) else: logging.info("RestartBot: Component %s Running" % component) # // # // Mail notification #// if len(restartedComponents) > 0 : mailMsg = "RestartBot. The following components were not running. The have been restarted:\n" for component in restartedComponents : mailMsg += " %s" % (component) mailMsg += "\nYours, RestartBot." self.mail(mailMsg) return
def getGroup(): """ Read LSF group name from config file Should I cache this ? """ try: config = loadProdAgentConfiguration() except StandardError, ex: msg = "Error reading configuration:\n" msg += str(ex) logging.error(msg) raise RuntimeError, msg
def __init__(self): self.pluginConfig = None self.prodAgentConfig = loadProdAgentConfiguration() try: # // # // Always searches in JobCreator Config Block #// for parameter called CreatorPluginConfig self.pluginConfig = loadPluginConfig("JobCreator", "Creator") except StandardError, ex: msg = "Failed to load Plugin Config for Creator Plugin:\n" msg += "Plugin Name: %s\n" % self.__class__.__name__ msg += str(ex) logging.warning(msg)
def main( argv ): dbName = "PromptCalibDB" try: opts, args = getopt.getopt(argv, "h", ["help"]) except getopt.GetoptError: print "Unable to parse argument list" usage() sys.exit(2) for o, a in opts: if o in ("-h", "--help"): usage() sys.exit() if len(args) > 1: print "Only expected one non-option argument" usage() sys.exit(2) elif len(args) == 1: dbName = args[0] paConfig = loadProdAgentConfiguration() t0astDBConfig = paConfig.getConfig( dbName ) print "Going to install the WMBS schema:" print " File: %s" % t0astDBConfig["dbName"] print "" print "Is this OK? (yes/no): ", line = sys.stdin.readline() if line != "yes\n": sys.exit(0) print "" clearDBInstance(t0astDBConfig["dbName"]) t0astDBConn = DBFactory.DBFactory(logging,"sqlite:///"+ t0astDBConfig['dbName']) dbInterface = t0astDBConn.connect() myThread = threading.currentThread() myThread.dialect = "sqlite" myThread.dbi = dbInterface myThread.logger = logging installWMBS(t0astDBConn, dbInterface) setupWMBS() sys.exit(0)
def __init__(self, cacheDepth = _DEFAULT_CACHE_DEPTH): self.depth = cacheDepth self.fileBackend = None self.t0astDBConn = None self.accessTimes = {} self.runConfigs = {} # The path to the Offline ConfDB file and the directory to cache # framework configs is stored in the RunConfig section of the ProdAgent # config file. paConfig = loadProdAgentConfiguration().getConfig("RunConfig") self.offlineConfDB = paConfig["OfflineConfDB"] self.configCache = paConfig["ConfigCacheDir"] logging.debug("RunConfigCache instantiated with depth=%s" % self.depth)
def verifySites(): """ _verifySites_ (Safely) Extract the VerifySites parameter from the JobQueue configuration block. Convert this to a True/False value and return it, default is True (verify sites against site DB) """ try: paConfig = loadProdAgentConfiguration() except Exception, ex: msg = "Unable to load PA Config: %s\n" % str(ex) msg += "VerifySites defaulting to True" logging.debug(msg) return True
def installPADetails(dashboardInfo): """ _installPADetails_ Add PA IDs & meta information to dashboard info """ prodAgentConfig = loadProdAgentConfiguration() paBlock = prodAgentConfig.get('ProdAgent', {}) prodMonBlock = prodAgentConfig.get("ProdMon", {}) paName = paBlock.get("ProdAgentName", "ProdAgent") team = prodMonBlock.get("Team", "NoTeam") dashboardInfo['ProdAgent'] = paName dashboardInfo['ProductionTeam'] = team return
def StartComponent(): """ _StartComponent_ Function that Starts this Component """ try: config = loadProdAgentConfiguration() componentConfig = config.getConfig('AlertHandler') except Exception, ex: msg = 'Failed to load ProdAgent Configuration' msg +='\nDetails: %s' % str(ex) raise RuntimeError, msg
def __init__(self): paConfig = loadProdAgentConfiguration() t0astDBConfig = paConfig.getConfig("Tier0DB") self.t0astDBConn = Tier0DB.Tier0DB(t0astDBConfig, manageGlobal = True) self.t0astDBConn.connect() localDBSConfig = paConfig.getConfig("LocalDBS") globalDBSConfig = paConfig.getConfig("GlobalDBSDLS") # phedexConfig = paConfig.getConfig("PhEDExConfig") self.localDBSUrl = localDBSConfig["DBSURL"] self.globalDBSUrl = globalDBSConfig["DBSURL"] # self.phedexDSUrl = phedexConfig["DataServiceURL"] self.localDbsApi = DbsApi({'url':localDBSConfig["DBSURL"]}) self.globalDbsApi = DbsApi({'url':globalDBSConfig["DBSURL"]})
def __init__(self): """ Overload constructor to add TaskQueue API initialization. """ BulkSubmitterInterface.__init__(self) logging.debug("<<<<< Creating TaskQueueSubmitter>>>>>") # Try to get the TaskQueue conf file from ProdAgentConfig.xml what = "TaskQueue" try: cfg = loadProdAgentConfiguration() self.tqConfig = cfg.getConfig(what) except StandardError, ex: msg = "%s.Config:" % what msg += "Unable to load ProdAgent Config for " + what msg += "%s\n" % ex logging.warning(msg)
def __init__(self, args): """ """ logging.debug("<<<<< Creating TaskQueueKiller>>>>>") # Try to get the TaskQueue conf file from ProdAgentConfig.xml what = "TaskQueue" try: cfg = loadProdAgentConfiguration() self.tqConfig = cfg.getConfig(what) except StandardError, ex: msg = "%s.Config:" % what msg += "Unable to load ProdAgent Config for " + what msg += "%s\n" % ex logging.critical(msg) raise ProdAgentException(msg)
def initialize(): global maxRetries, requestJobFactory, jobSpecDir if not requestJobFactory: try: config = loadProdAgentConfiguration() compCfg = config.getConfig("ProdMgrInterface") jobSpecDir=compCfg['JobSpecDir'] jobStatesCfg = config.getConfig("JobStates") if jobStatesCfg.has_key('maxRetries'): maxRetries=jobStatesCfg['maxRetries'] logging.debug("Initializing RequestJobFactory") requestJobFactory = RequestJobFactory(None,'',0) requestJobFactory.workflowSpec = WorkflowSpec() requestJobFactory.workingDir = compCfg['JobSpecDir'] logging.debug("Writing job cut specs to: "+str(jobSpecDir)) except StandardError, ex: msg = "Error reading configuration:\n" msg += str(ex) logging.info("WARNING: "+msg)
def configureScheduler(self, schedSession) : """ _configureScheduler_ perform any scheduler specific operation Specific implementation in the Scheduler specific part (e.g. BlGLiteBulkSubmitter) """ from ProdAgentCore.Configuration import loadProdAgentConfiguration config = loadProdAgentConfiguration() compCfg = config.getConfig("JobSubmitter") filename = os.path.join(compCfg['ComponentDir'], 'lastDelegation') format = '%Y-%m-%d %H:%M:%S' timestamp = 0 if os.path.exists(filename) : tsfile = open(filename, 'r') tsentry = tsfile.read().strip() try : timestamp = time.time() - \ time.mktime(time.strptime(tsentry, format)) timestamp /= 43200 except ValueError: pass tsfile.close() if timestamp > 0 and timestamp < 1: return logging.info('delegating proxy to wms after %s hours' % \ (timestamp * 12)) schedSession.getSchedulerInterface().delegateProxy() tsfile = open(filename, 'w') logging.info ('%s' % time.strftime(format)) tsfile.write(time.strftime(format)) tsfile.close() return
def loadPluginConfig(component, plugin): """ _loadPluginConfig_ Get the PluginConfiguration object for the component plugin using the name specified. The Component will be looked up from the main ProdAgentConfig using the PRODAGENT_CONFIG env var. This Config will be searched for the plugin file using the convention: <pluginName>PluginConfig This is assumed to be a file which is a saved PluginConfiguration object """ prodAgentConfig = loadProdAgentConfiguration() compConfig = prodAgentConfig.getConfig(component) keyname = "%sPluginConfig" % plugin filename = compConfig.get(keyname, None) if filename == None: msg = "Cannot find entry for Plugin: %s\n" % plugin msg += "In Config for component: %s\n" % component msg += "Expected parameter: %s\n " % keyname raise RuntimeError, msg filename = os.path.expandvars(filename) if not os.path.exists(filename): msg = "Plugin Configuration file Not Found:\n" msg += "%s\n" % filename msg += "For component %s plugin %s\n" % (component, plugin) raise RuntimeError, msg plugCfg = PluginConfiguration() try: plugCfg.loadFromFile(filename) except StandardError, ex: msg = "Error loading Plugin Configuration for:\n" msg += "Component %s Plugin %s\n" % (component, plugin) msg += "From File:\n%s\n" % filename msg += str(ex) raise RuntimeError, msg
def index(self, componentName) : config = loadProdAgentConfiguration() componentDir = config.getConfig(componentName)['ComponentDir'] componentDir = os.path.expandvars(componentDir) logs = [] for file in os.listdir(componentDir) : if file.find('ComponentLog') > -1 : logs.append(file) logs.sort() html = """<html><body><h2>List of Available Logs for Component %s </h2>\n """ % componentName if len(logs) == 0 : html = """<html><body><h2>No Available Logs for Component %s! </h2>\n """ % componentName else : html += "<ul>\n" for log in logs: logFile = os.path.join(componentName,log) html += "<li><a href=\"%s?logFile=%s\">%s</a></li>\n" % ( self.writelog, logFile, log) html += "</ul>\n" html += """</body></html>""" return html
def __init__(self, config): """ __init__ """ self.jobavaliableflag = False self.tarPath = config.PilotManagerComponent.tarPath self.pilotcodeDir = config.PilotManagerComponent.pilotCode self.tqServer = config.PilotManagerComponent.tqAddress self.emulationMode = config.PilotManagerComponent.emulationMode self.pilotParams = None try: cfg = loadProdAgentConfiguration() pilot = cfg.getConfig("Pilot") self.pilotParams = pilot except: logging.debug("could not load PRODAGENT_CONFIG file") if ( not self.pilotParams ): self.pilotParams={'badAttempts':6,'noTaskAttempts':6} Harness.__init__(self, config)
def __init__(self, argv): """ initialize test instance """ # initialize super class super(jobUnitTests, self).__init__(argv) # read configuration self.configure() # MySQL: get DB configuration from config file if self.database == "MySQL": from ProdAgentCore.Configuration import loadProdAgentConfiguration cfg = loadProdAgentConfiguration() dbConfig = cfg.getConfig("ProdAgentDB") dbConfig['dbName'] += '_BOSS' # create DB instance self.dbInstance = MysqlInstance(dbConfig) else: dbConfig = {} dbConfig['dbName'] = 'ProdAgentDB_BOSS' # create DB instance self.dbInstance = SqliteInstance(dbConfig) # remove all information from database session = SafeSession(dbInstance = self.dbInstance) try: session.execute("delete from bl_runningjob") session.execute("delete from bl_job") session.execute("delete from bl_task") session.commit() except DbError, msg: print "Cannot erase database contents: " + str(msg) sys.exit(1)
def __init__(self, argv): """ initialize test instance """ # initialize super class super(jobUnitTests, self).__init__(argv) # read configuration self.configure() # MySQL: get DB configuration from config file if self.database == "MySQL": from ProdAgentCore.Configuration import loadProdAgentConfiguration cfg = loadProdAgentConfiguration() dbConfig = cfg.getConfig("ProdAgentDB") dbConfig['dbName'] += '_BOSS' # create DB instance self.dbInstance = MysqlInstance(dbConfig) else: dbConfig = {} dbConfig['dbName'] = 'ProdAgentDB_BOSS' # create DB instance self.dbInstance = SqliteInstance(dbConfig) # remove all information from database session = SafeSession(dbInstance=self.dbInstance) try: session.execute("delete from bl_runningjob") session.execute("delete from bl_job") session.execute("delete from bl_task") session.commit() except DbError, msg: print "Cannot erase database contents: " + str(msg) sys.exit(1)
def index(self): config = loadProdAgentConfiguration() components = config.listComponents() html = """<head>\n""" html += "<meta http-equiv=\"refresh\" content=\"60\" />\n\n" html += """</head>""" html += """<html><body><h2>ProdAgent Component Status</h2>\n """ html += "<table>\n" html += "<tr><th>Component</th><th>Status</th><th>Logs</th><th>PostMortens</th></tr>\n" for component in components : componentDir = config.getConfig(component)['ComponentDir'] componentDir = os.path.expandvars(componentDir) daemonXML = os.path.join(componentDir, "Daemon.xml") daemon = DaemonDetails(daemonXML) if not daemon.isAlive(): html += "<tr><td>%s</td><td><font color=\"FF0000\">Not Running</font></td>" % component else : html += "<tr><td>%s</td><td>PID : %s</td>" % (component, daemon['ProcessID']) html += "<td><a href=\"%s?componentName=%s\">Show</a></td>\n" % (self.complog, component) html += "<td><a href=\"%s?componentName=%s\">Show</a></td></tr>\n" % (self.postmorten, component) html += "</table>\n" html += """</body></html>""" return html