def setUp(self): """ _setUp_ Setup the database and logging connection. """ logging.basicConfig() self.logger = logging.getLogger('LogDBTest') # self.logger.setLevel(logging.DEBUG) self.schema = [] self.couchApps = ["LogDB"] self.testInit = TestInitCouchApp('LogDBTest') self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=self.schema, useDefault=False) dbName = 'logdb_t' self.testInit.setupCouch(dbName, *self.couchApps) url = "%s/%s" % (self.testInit.couchUrl, dbName) identifier = 'agentname' self.agent_inst = LogDB(url, identifier, logger=self.logger, thread_name="MainThread") self.agent_inst2 = LogDB(url, identifier, logger=self.logger, thread_name="Test") identifier = '/DC=org/DC=doegrids/OU=People/CN=First Last 123' self.user_inst = LogDB(url, identifier, logger=self.logger, thread_name="MainThread") self.report = LogDBReport(self.agent_inst)
def setUp(self): """ _setUp_ Setup the database and logging connection. """ logging.basicConfig() self.logger = logging.getLogger('LogDBTest') # self.logger.setLevel(logging.DEBUG) url = os.environ.get('COUCHURL', 'http://localhost:5984') + '/logdb_t' identifier = 'agentname' self.agent_inst = LogDB(url, identifier, logger=self.logger, create=True) self.agent_inst2 = LogDB(url, identifier, logger=self.logger, create=True, thread_name="Test") identifier = '/DC=org/DC=doegrids/OU=People/CN=First Last 123' self.user_inst = LogDB(url, identifier, logger=self.logger, create=True) self.report = LogDBReport(self.agent_inst)
def setUpLogDB(self, config): if hasattr(config, "central_logdb_url"): # default set up for logDB config section need to contain propervalues from WMCore.Services.LogDB.LogDB import LogDB self.logDB = LogDB(config.central_logdb_url, config.log_reporter, thread_name=config.object.rsplit(".", 1)[-1]) else: self.logDB = None
def logDBCleanUp(self, config): """ gather active data statistics """ logdb = LogDB(config.central_logdb_url, config.log_reporter) logdb.cleanup(config.logDBCleanDuration) self.logger.info("cleaned up log db") return
def setUpLogDB(self, myThread): # setup logDB if hasattr(self.component.config, "General") and \ hasattr(self.component.config.General, "central_logdb_url") and \ hasattr(self.component.config, "Agent"): from WMCore.Services.LogDB.LogDB import LogDB myThread.logdbClient = LogDB(self.component.config.General.central_logdb_url, self.component.config.Agent.hostName, logger=logging) else: myThread.logdbClient = None return
def __init__(self, **kwargs): if not kwargs.get('logger'): import logging kwargs['logger'] = logging self.logger = kwargs['logger'] # this will break all in one test self.reqMgr2 = ReqMgr(kwargs.get("reqmgr2_endpoint", None)) centralurl = kwargs.get("central_logdb_url", "") identifier = kwargs.get("log_reporter", "") # set the thread name before creat the log db. # only sets that when it is not set already myThread = threading.currentThread() if myThread.getName() == "MainThread": myThread.setName(self.__class__.__name__) self.logdb = LogDB(centralurl, identifier, logger=self.logger) self.previous_state = {}
def advanceStatus(self, config): """ Advance the request status based on the global workqueue elements status """ reqmgrSvc = ReqMgr(config.reqmgr2_url, logger=self.logger) gqService = WorkQueue(config.workqueue_url) wmstatsSvc = WMStatsServer(config.wmstats_url, logger=self.logger) logdb = LogDB(config.central_logdb_url, config.log_reporter) self.logger.info("Getting GQ data for status check") wfStatusDict = gqService.getWorkflowStatusFromWQE() self.logger.info("Advancing status") moveForwardStatus(reqmgrSvc, wfStatusDict, self.logger) moveToCompletedForNoWQJobs(reqmgrSvc, wfStatusDict, self.logger) moveToArchived(wmstatsSvc, reqmgrSvc, logdb, config.archiveDelayHours, self.logger) self.logger.info("Done advancing status") return
def __call__(self, parameters): """ Thread entry point; handles synchronisation with run and terminate conditions """ try: msg = "Initialising worker thread %s" % str(self) logging.info(msg) # Call thread startup method self.initInThread(parameters) msg = "Worker thread %s started" % str(self) logging.info(msg) # heartbeat needed to be called after self.initInThread # to get the right name myThread = threading.currentThread() if hasattr(self.component.config, "Agent"): self.useHeartbeat = getattr(self.component.config.Agent, "useHeartbeat", True) self.workerName = myThread.getName() if hasattr(self.component.config, "General") and \ hasattr(self.component.config.General, "central_logdb_url") and \ hasattr(self.component.config, "Agent"): from WMCore.Services.LogDB.LogDB import LogDB myThread.logdbClient = LogDB( self.component.config.General.central_logdb_url, self.component.config.Agent.hostName, logger=logging) else: myThread.logdbClient = None if self.useHeartbeat: self.heartbeatAPI.updateWorkerHeartbeat(self.workerName) # Run event loop while termination is not flagged while not self.notifyTerminate.isSet(): # Check manager hasn't paused threads if self.notifyPause.isSet(): self.notifyResume.wait() else: # Catch case where threads were paused and then terminated # - threads should not run in this case! if not self.notifyTerminate.isSet(): # Do some work! try: try: # heartbeat needed to be called after self.initInThread # to get the right name if self.useHeartbeat: self.heartbeatAPI.updateWorkerHeartbeat( self.workerName, "Running") except Exception as ex: msg = " Failed to update heartbeat for worker %s" % str( self) msg += ":\n %s" % str(ex) msg += "\n Skipping worker algorithm!" logging.error(msg) else: self.algorithm(parameters) # Catch if someone forgets to commit/rollback if myThread.transaction.transaction is not None: msg = """ Thread %s: Transaction reached end of poll loop.""" % self.workerName msg += " Raise a bug against me. Rollback." logging.error(msg) myThread.transaction.rollback() except Exception as ex: if myThread.transaction.transaction is not None: myThread.transaction.rollback() msg = "Error in worker algorithm (1):\nBacktrace:\n " msg += (" %s %s" % (str(self), str(ex))) stackTrace = traceback.format_tb( sys.exc_info()[2], None) for stackFrame in stackTrace: msg += stackFrame logging.error(msg) # force entire component to terminate try: self.component.prepareToStop() except Exception as ex: logging.error( "Failed to halt component after worker crash: %s", str(ex)) if self.useHeartbeat: self.heartbeatAPI.updateWorkerError( self.workerName, msg) raise ex # Put the thread to sleep self.sleepThread() # Call specific thread termination method self.terminate(parameters) except Exception as ex: # Notify error msg = "Error in event loop (2): %s %s\nBacktrace:\n" msg = msg % (str(self), str(ex)) stackTrace = traceback.format_tb(sys.exc_info()[2], None) for stackFrame in stackTrace: msg += stackFrame logging.error(msg) # Indicate to manager that thread is done self.terminateCallback(threading.currentThread().name) # All done msg = "Worker thread %s terminated" % str(self) logging.info(msg)
def __init__(self, msConfig, logger=None): """ Runs the basic setup and initialization for the MSRuleCleaner module :param msConfig: micro service configuration """ super(MSRuleCleaner, self).__init__(msConfig, logger=logger) self.msConfig.setdefault("verbose", True) self.msConfig.setdefault("interval", 60) self.msConfig.setdefault("services", ['ruleCleaner']) self.msConfig.setdefault("rucioWmaAccount", "wma_test") self.msConfig.setdefault("rucioMStrAccount", "wmcore_transferor") self.msConfig.setdefault('enableRealMode', False) self.mode = "RealMode" if self.msConfig['enableRealMode'] else "DryRunMode" self.curlMgr = RequestHandler() self.targetStatusRegex = re.compile(r'.*archived') self.logDB = LogDB(self.msConfig["logDBUrl"], self.msConfig["logDBReporter"], logger=self.logger) self.wmstatsSvc = WMStatsServer(self.msConfig['wmstatsUrl'], logger=self.logger) # Building all the Pipelines: pName = 'plineMSTrCont' self.plineMSTrCont = Pipeline(name=pName, funcLine=[Functor(self.setPlineMarker, pName), Functor(self.setParentDatasets), Functor(self.getRucioRules, 'container', self.msConfig['rucioMStrAccount']), Functor(self.cleanRucioRules)]) pName = 'plineMSTrBlock' self.plineMSTrBlock = Pipeline(name=pName, funcLine=[Functor(self.setPlineMarker, pName), Functor(self.setParentDatasets), Functor(self.getRucioRules, 'block', self.msConfig['rucioMStrAccount']), Functor(self.cleanRucioRules)]) pName = 'plineAgentCont' self.plineAgentCont = Pipeline(name=pName, funcLine=[Functor(self.setPlineMarker, pName), Functor(self.getRucioRules, 'container', self.msConfig['rucioWmaAccount']), Functor(self.cleanRucioRules)]) pName = 'plineAgentBlock' self.plineAgentBlock = Pipeline(name=pName, funcLine=[Functor(self.setPlineMarker, pName), Functor(self.getRucioRules, 'block', self.msConfig['rucioWmaAccount']), Functor(self.cleanRucioRules)]) pName = 'plineArchive' self.plineArchive = Pipeline(name=pName, funcLine=[Functor(self.setPlineMarker, pName), Functor(self.findTargetStatus), Functor(self.setClean), Functor(self.setArchivalDelayExpired), Functor(self.setLogDBClean), Functor(self.archive)]) # Building the different set of plines we will need later: # NOTE: The following are all the functional pipelines which are supposed to include # a cleanup function and report cleanup status in the MSRuleCleanerWflow object self.cleanuplines = [self.plineMSTrCont, self.plineMSTrBlock, self.plineAgentCont, self.plineAgentBlock] # Building an auxiliary list of cleanup pipeline names only: self.cleanupPipeNames = [pline.name for pline in self.cleanuplines] # Building lists of pipelines related only to Agents or MStransferror self.agentlines = [self.plineAgentCont, self.plineAgentBlock] self.mstrlines = [self.plineMSTrCont, self.plineMSTrBlock] # Initialization of the 'cleaned' and 'archived' counters: self.wfCounters = {'cleaned': {}, 'archived': {'normalArchived': 0, 'forceArchived': 0}} self.globalLocks = set()
def __init__(self, app, config, mount): self.base = config.base self.rootdir = '/'.join(WMCore.__file__.split('/')[:-1]) if config and not isinstance(config, dict): web_config = config.dictionary_() if not config: web_config = {'base': self.base} TemplatedPage.__init__(self, web_config) imgdir = os.environ.get('RM_IMAGESPATH', os.getcwd() + '/images') self.imgdir = web_config.get('imgdir', imgdir) cssdir = os.environ.get('RM_CSSPATH', os.getcwd() + '/css') self.cssdir = web_config.get('cssdir', cssdir) jsdir = os.environ.get('RM_JSPATH', os.getcwd() + '/js') self.jsdir = web_config.get('jsdir', jsdir) spdir = os.environ.get('RM_SPECPATH', os.getcwd() + '/specs') self.spdir = web_config.get('spdir', spdir) # read scripts area and initialize data-ops scripts self.sdir = os.environ.get('RM_SCRIPTS', os.getcwd() + '/scripts') self.sdir = web_config.get('sdir', self.sdir) self.sdict_thr = web_config.get('sdict_thr', 600) # put reasonable 10 min interval self.sdict = {'ts': time.time()} # placeholder for data-ops scripts self.update_scripts(force=True) # To be filled at run time self.cssmap = {} self.jsmap = {} self.imgmap = {} self.yuimap = {} std_specs_dir = os.path.join(self.rootdir, 'WMSpec/StdSpecs') self.std_specs = spec_list(std_specs_dir) self.std_specs.sort() # Update CherryPy configuration mime_types = ['text/css'] mime_types += [ 'application/javascript', 'text/javascript', 'application/x-javascript', 'text/x-javascript' ] cherryconf.update({ 'tools.encode.on': True, 'tools.gzip.on': True, 'tools.gzip.mime_types': mime_types, }) self._cache = {} # initialize access to reqmgr2 APIs self.reqmgr_url = config.reqmgr.reqmgr2_url self.reqmgr = ReqMgr(self.reqmgr_url) # only gets current view (This might cause to reponse time much longer, # If upto date view is not needed overwrite Fale) self.reqmgr._noStale = True # get fields which we'll use in templates cdict = config.reqmgr.dictionary_() self.couch_url = cdict.get('couch_host', '') self.couch_dbname = cdict.get('couch_reqmgr_db', '') self.couch_wdbname = cdict.get('couch_workload_summary_db', '') self.acdc_url = cdict.get('acdc_host', '') self.acdc_dbname = cdict.get('acdc_db', '') self.configcache_url = cdict.get('couch_config_cache_url', self.couch_url) self.dbs_url = cdict.get('dbs_url', '') self.dqm_url = cdict.get('dqm_url', '') self.sw_ver = cdict.get('default_sw_version', 'CMSSW_7_6_1') self.sw_arch = cdict.get('default_sw_scramarch', 'slc6_amd64_gcc493') # LogDB holder centralurl = cdict.get("central_logdb_url", "") identifier = cdict.get("log_reporter", "reqmgr2") self.logdb = LogDB(centralurl, identifier) # local team cache which will request data from wmstats base, uri = self.reqmgr_url.split('://') base_url = '%s://%s' % (base, uri.split('/')[0]) self.wmstatsurl = cdict.get('wmstats_url', '%s/wmstatsserver' % base_url) if not self.wmstatsurl: raise Exception( 'ReqMgr2 configuration file does not provide wmstats url') self.team_cache = [] # fetch assignment arguments specification from StdBase self.assignArgs = StdBase().getWorkloadAssignArgs() self.assignArgs = { key: val['default'] for key, val in self.assignArgs.items() }
def __init__(self, rest, config): super(LogDBTasks, self).__init__(config) self.logdb = LogDB(config.central_logdb_url, config.log_reporter)
def __init__(self, app, config, mount): self.base = config.base self.rootdir = '/'.join(WMCore.__file__.split('/')[:-1]) if config and not isinstance(config, dict): web_config = config.dictionary_() if not config: web_config = {'base': self.base} TemplatedPage.__init__(self, web_config) imgdir = os.environ.get('RM_IMAGESPATH', os.getcwd() + '/images') self.imgdir = web_config.get('imgdir', imgdir) cssdir = os.environ.get('RM_CSSPATH', os.getcwd() + '/css') self.cssdir = web_config.get('cssdir', cssdir) jsdir = os.environ.get('RM_JSPATH', os.getcwd() + '/js') self.jsdir = web_config.get('jsdir', jsdir) spdir = os.environ.get('RM_SPECPATH', os.getcwd() + '/specs') self.spdir = web_config.get('spdir', spdir) # read scripts area and initialize data-ops scripts self.sdir = os.environ.get('RM_SCRIPTS', os.getcwd() + '/scripts') self.sdir = web_config.get('sdir', self.sdir) self.sdict_thr = web_config.get('sdict_thr', 600) # put reasonable 10 min interval self.sdict = {'ts': time.time()} # placeholder for data-ops scripts self.update_scripts(force=True) # To be filled at run time self.cssmap = {} self.jsmap = {} self.imgmap = {} self.yuimap = {} std_specs_dir = os.path.join(self.rootdir, 'WMSpec/StdSpecs') self.std_specs = spec_list(std_specs_dir, 'WMSpec.StdSpecs') self.std_specs.sort() # Update CherryPy configuration mime_types = ['text/css'] mime_types += [ 'application/javascript', 'text/javascript', 'application/x-javascript', 'text/x-javascript' ] cherryconf.update({ 'tools.encode.on': True, 'tools.gzip.on': True, 'tools.gzip.mime_types': mime_types, }) self._cache = {} # initialize rest API statedir = '/tmp' app = RESTMain(config, statedir) # REST application mount = '/rest' # mount point for cherrypy service api = RestApiHub(app, config.reqmgr, mount) # initialize access to reqmgr2 APIs self.reqmgr = ReqMgr(config.reqmgr.reqmgr2_url) # only gets current view (This might cause to reponse time much longer, # If upto date view is not needed overwrite Fale) self.reqmgr._noStale = True # admin helpers self.admin_info = Info(app, api, config.reqmgr, mount=mount + '/info') self.admin_group = Group(app, api, config.reqmgr, mount=mount + '/group') self.admin_team = Team(app, api, config.reqmgr, mount=mount + '/team') # get fields which we'll use in templates cdict = config.reqmgr.dictionary_() self.couch_url = cdict.get('couch_host', '') self.couch_dbname = cdict.get('couch_reqmgr_db', '') self.couch_wdbname = cdict.get('couch_workload_summary_db', '') self.acdc_url = cdict.get('acdc_host', '') self.acdc_dbname = cdict.get('acdc_db', '') self.configcache_url = cdict.get('couch_config_cache_url', self.couch_url) self.dbs_url = cdict.get('dbs_url', '') self.dqm_url = cdict.get('dqm_url', '') self.sw_ver = cdict.get('default_sw_version', 'CMSSW_5_2_5') self.sw_arch = cdict.get('default_sw_scramarch', 'slc5_amd64_gcc434') # LogDB holder centralurl = cdict.get("central_logdb_url", "") identifier = cdict.get("log_reporter", "reqmgr2") self.logdb = LogDB(centralurl, identifier)