def preInitialization(self): """ Add required worker modules to work threads """ # use a factory to dynamically load handlers. factory = WMFactory('generic') self.messages['AddWorkflowToManage'] = \ factory.loadObject(\ "WMComponent.WorkflowManager.Handler.AddWorkflowToManage", self) self.messages['RemoveWorkflowFromManagement'] = \ factory.loadObject(\ "WMComponent.WorkflowManager.Handler.RemoveWorkflowFromManagement", self) self.messages['AddToWorkflowManagementLocationList'] = \ factory.loadObject(\ "WMComponent.WorkflowManager.Handler.AddToWorkflowManagementLocationList", self) self.messages['RemoveFromWorkflowManagementLocationList'] = \ factory.loadObject(\ "WMComponent.WorkflowManager.Handler.RemoveFromWorkflowManagementLocationList", self) # Add event loop to worker manager myThread = threading.currentThread() pollInterval = self.config.WorkflowManager.pollInterval logging.info("Setting poll interval to %s seconds" % pollInterval) myThread.workerThreadManager.addWorker(WorkflowManagerPoller(), \ pollInterval)
class Plotter(RESTModel): ''' A class that generates a plot object to send to the formatter, possibly downloading the data from a remote resource along the way ''' def __init__(self, config): RESTModel.__init__(self, config) self.methods['POST'] = {'plot': {'version':1, 'call':self.plot, 'args': ['type', 'data', 'url'], 'expires': 300}} self.methods['GET'] = {'plot': {'version':1, 'call':self.plot, 'args': ['type', 'data', 'url'], 'expires': 300}, 'doc': {'version':1, 'call':self.doc, 'args': ['type'], 'expires':300}} self.factory = WMFactory('plot_fairy', 'WMCore.HTTPFrontEnd.PlotFairy.Plots') def plot(self, *args, **kwargs): input = self.sanitise_input(args, kwargs, 'plot') plot = self.factory.loadObject(input['type']) return {'figure': plot(input['data'])} def doc(self, *args, **kwargs): input = self.sanitise_input(args, kwargs, 'doc') plot = self.factory.loadObject(input['type']) return {'doc': plot.doc()} def validate_input(self, input, verb, method): assert 'type' in input.keys(), "no type provided - what kind of plot do you want?" if method=='doc': return input valid_data = {} assert 'data' in input.keys() or 'url' in input.keys(), "neither data nor url provided - please provide at least one" if 'url' in input.keys(): match = URL_REGEX.match(input['url']) assert match != None, "`%s' is not a valid URL" % input['url'] host = match.group(1) + '://' + match.group(3) if match.group(4): host += match.group(4) uri = match.group(5) result,status,reason,fromcache = JSONRequests(host).get(uri) valid_data.update(result) if 'data' in input.keys(): valid_data.update(json.loads(urllib.unquote(input['data']))) return {'data':valid_data,'type':input['type']}
def __init__(self, config, dbsconfig=None): """ Initialise class members """ myThread = threading.currentThread() BaseWorkerThread.__init__(self) self.config = config # This is slightly dangerous, but DBSUpload depends # on DBSInterface anyway self.bufferFactory = DAOFactory( package="WMComponent.DBSBuffer.Database", logger=myThread.logger, dbinterface=myThread.dbi) factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database.Interface") self.uploadToDBS = factory.loadObject("UploadToDBS") addFactory = WMFactory("dbsBuffer", "WMComponent.DBSBuffer.Database.Interface") self.addToBuffer = addFactory.loadObject("AddToBuffer") # Set DBSInterface self.dbsInterface = DBSInterface(config=config) # Set DAOs self.setBlock = self.bufferFactory(classname="DBSBufferFiles.SetBlock") self.setStatus = self.bufferFactory( classname="DBSBufferFiles.SetStatus") # Set config parameters self.maxBlockFiles = self.config.DBSInterface.DBSBlockMaxFiles self.maxBlockTime = self.config.DBSInterface.DBSBlockMaxTime self.maxBlockSize = self.config.DBSInterface.DBSBlockMaxSize self.doMigration = getattr(self.config.DBSInterface, 'doGlobalMigration', True) logging.debug( "Initializing with maxFiles %i, maxTime %i, maxSize %i" % (self.maxBlockFiles, self.maxBlockTime, self.maxBlockSize)) if dbsconfig == None: self.dbsconfig = config # initialize the alert framework (if available - config.Alert present) # self.sendAlert will be then be available self.initAlerts(compName="DBSUpload") return
def __init__(self, config, dbsconfig = None): """ Initialise class members """ myThread = threading.currentThread() BaseWorkerThread.__init__(self) self.config = config # This is slightly dangerous, but DBSUpload depends # on DBSInterface anyway self.bufferFactory = DAOFactory(package = "WMComponent.DBSBuffer.Database", logger = myThread.logger, dbinterface = myThread.dbi) factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database.Interface") self.uploadToDBS = factory.loadObject("UploadToDBS") addFactory = WMFactory("dbsBuffer", "WMComponent.DBSBuffer.Database.Interface") self.addToBuffer = addFactory.loadObject("AddToBuffer") # Set DBSInterface self.dbsInterface = DBSInterface(config = config) # Set DAOs self.setBlock = self.bufferFactory(classname = "DBSBufferFiles.SetBlock") self.setStatus = self.bufferFactory(classname = "DBSBufferFiles.SetStatus") # Set config parameters self.maxBlockFiles = self.config.DBSInterface.DBSBlockMaxFiles self.maxBlockTime = self.config.DBSInterface.DBSBlockMaxTime self.maxBlockSize = self.config.DBSInterface.DBSBlockMaxSize self.doMigration = getattr(self.config.DBSInterface, 'doGlobalMigration', True) logging.debug("Initializing with maxFiles %i, maxTime %i, maxSize %i" % (self.maxBlockFiles, self.maxBlockTime, self.maxBlockSize)) if dbsconfig == None: self.dbsconfig = config # initialize the alert framework (if available - config.Alert present) # self.sendAlert will be then be available self.initAlerts(compName = "DBSUpload") return
def __init__(self, options, logger, defaults): self.defaults = defaults self.logger = logger # Format the site name self.site = options.site.lower() self.node = options.site.replace('_Buffer', '_MSS') if not self.node.endswith('_MSS'): self.node += '_MSS' # Connect to local couch self.localcouch = CouchServer(options.localdb) # Attempt to load the configuration self.configdb = self.localcouch.connectDatabase('%s_configuration' % self.site) self.load_config() # Now update configuration with command line parameters or defaults for key in defaults: storeName = defaults[key]['opts']['dest'] # Overwrite those options passed on the command line cmdOption = getattr(options, storeName) if cmdOption: print "Updating config", storeName, cmdOption self.config[storeName] = cmdOption # Use defaults for any options which remain unset if not self.config.has_key(storeName): print "Using default config", storeName, defaults[key]['opts']['default'] self.config[storeName] = defaults[key]['opts']['default'] # Check waittime is sensible if self.config['waittime'] < 31: self.config['waittime'] = 31 # Save the active configuration self.save_config() # Finish up remote DB connection, and other config self.logger.info('local databases: %s' % self.localcouch) #Create our stager factory = WMFactory('stager_factory', 'StageManager.Agents.Stagers') queuedb = self.localcouch.connectDatabase('%s_stagequeue' % self.site) statsdb = self.localcouch.connectDatabase('%s_statistics' % self.site) requestdb = self.localcouch.connectDatabase('%s_requests' % self.site) # Parse stager options sopts = self.config['stageroptions'].split(",") def OptFilter(opt): return opt.find("=") > 0 sopts = filter(OptFilter, sopts) stagerOptions = {} for sopt in sopts: tokens = sopt.split("=") stagerOptions[tokens[0]] = tokens[1] self.stager = factory.loadObject(self.config['stagerplugin'], args=[queuedb, statsdb, self.configdb, requestdb, stagerOptions, self.logger, self.site], listFlag = True)
def preInitialization(self): """ __preInitialization__ """ if not hasattr(self.config.PilotManagerComponent, "pilotManagerHandler"): self.config.PilotManagerComponent.pilotManagerHandler = 'PilotManager.Handler.PilotManagerHandler' factory = WMFactory('PilotManager'); self.messages["NewPilotJob"] = factory.loadObject(\ self.config.PilotManagerComponent.pilotManagerHandler, self) self.messages["SubmitPilotJob"] = factory.loadObject(\ self.config.PilotManagerComponent.pilotManagerHandler, self) #create pilot tar file self.createPilotTar()
def setSchema(self, modules = [], params = None): """ Creates the schema in the database based on the modules input. This method needs to have been preceded by the setDatabaseConnection. """ myThread = threading.currentThread() parameters = None flag = False #Set up for typical DBCreator format: logger, dbi, params if params != None: parameters = [None, None, params] flag = True myThread.transaction.begin() for factoryName in modules: # need to create these tables for testing. # notice the default structure: <dialect>/Create factory = WMFactory(factoryName, factoryName + "." + myThread.dialect) create = factory.loadObject("Create", args = parameters, listFlag = flag) createworked = create.execute(conn = myThread.transaction.conn, transaction = myThread.transaction) if createworked: logging.debug("Tables for "+ factoryName + " created") else: logging.debug("Tables " + factoryName + " could not be created.") myThread.transaction.commit()
def setup(self, parameters): """ set db connection(couchdb, wmbs) to prepare to gather information """ # self.localQueue = WorkQueueService(self.config.AnalyticsDataCollector.localQueueURL) # set the connection for local couchDB call self.localCouchDB = LocalCouchDBData(self.config.AnalyticsDataCollector.localCouchURL, self.summaryLevel) # interface to WMBS/BossAir db myThread = threading.currentThread() # set wmagent db data self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi, myThread.logger) # set the connection for local couchDB call self.localSummaryCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.localWMStatsURL) logging.info("Setting the replication to central monitor ...") self.localSummaryCouchDB.replicate(self.config.AnalyticsDataCollector.centralWMStatsURL) self.centralWMStatsCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.centralWMStatsURL) if self.pluginName != None: pluginFactory = WMFactory("plugins", "WMComponent.AnalyticsDataCollector.Plugins") self.plugin = pluginFactory.loadObject(classname = self.pluginName)
def pollExternal(self, fileset): """ Call relevant external source and get file details """ logging.debug("Feeder name %s" % (fileset.name).split(":")[1]) try: factory = WMFactory("default", \ "WMCore.WMBSFeeder." + (fileset.name).split(":")[1]) feeder = factory.loadObject(classname="Feeder", getFromCache=False) feeder(fileset) except Exception as e: msg = "Feeder plugin \'%s\' unknown" \ % (fileset.name).split(":")[1] logging.info(msg) logging.info(e) logging.info(traceback.format_exc()) logging.info("aborting poll for...closing fileset") fileset.markOpen(False) fileset.commit() return fileset
def pollExternal(self, fileset): """ Call relevant external source and get file details """ logging.debug("Feeder name %s" % (fileset.name).split(":")[1]) try: factory = WMFactory("default", \ "WMCore.WMBSFeeder." + (fileset.name).split(":")[1]) feeder = factory.loadObject( classname = "Feeder", getFromCache = False ) feeder(fileset) except Exception as e : msg = "Feeder plugin \'%s\' unknown" \ % (fileset.name).split(":")[1] logging.info(msg) logging.info(e) logging.info( traceback.format_exc() ) logging.info("aborting poll for...closing fileset") fileset.markOpen(False) fileset.commit() return fileset
def testF_DBSUploadQueueSizeCheckForAlerts(self): """ Test will not trigger a real alert being sent unless doing some mocking of the methods used during DBSUploadPoller.algorithm() -> DBSUploadPoller.uploadBlocks() method. As done here, it probably can't be deterministic, yet the feature shall be checked. """ sizeLevelToTest = 1 myThread = threading.currentThread() config = self.createConfig() # threshold / value to check config.DBSUpload.alertUploadQueueSize = sizeLevelToTest # without this uploadBlocks method returns immediately name = "ThisIsATest_%s" % (makeUUID()) tier = "RECO" nFiles = sizeLevelToTest + 1 files = self.getFiles(name=name, tier=tier, nFiles=nFiles) datasetPath = "/%s/%s/%s" % (name, name, tier) # load components that are necessary to check status # (this seems necessary, else some previous tests started failing) factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database.Interface") dbinterface = factory.loadObject("UploadToDBS") dbsInterface = DBSInterface(config=config) localAPI = dbsInterface.getAPIRef() globeAPI = dbsInterface.getAPIRef(globalRef=True) testDBSUpload = DBSUploadPoller(config) # this is finally where the action (alert) should be triggered from testDBSUpload.algorithm() return
class GeneratorFactory(WMFactory): """ _GeneratorFactory_ Instantiate a WMFactory instance with the appropriate namespace """ def __init__(self): self.factory = WMFactory(self.__class__.__name__, "WMCore.JobSplitting.Generators") def getGenerator(self, generatorName, **options): """ _getGenerator_ factory method to return a step template instance based on the name of the step """ args = {} args.update(options) try: return self.factory.loadObject(generatorName, args) except WMException, wmEx: msg = "GeneratorFactory Unable to load Object: %s" % generatorName raise GeneratorFactoryException(msg) except Exception, ex: msg = "Error creating object %s in GeneratorFactory:\n" % generatorName msg += str(ex) raise GeneratorFactoryException(msg)
def preInitialization(self): print "JobMaker.preInitialization" # use a factory to dynamically load handlers. factory = WMFactory('generic') self.messages['MakeJob'] = \ factory.loadObject('WMCore.WMSpec.Makers.Handlers.MakeJob', self)
def setup(self, parameters): """ set db connection(couchdb, wmbs) to prepare to gather information """ # set the connection to local queue if not hasattr(self.config, "Tier0Feeder"): self.localQueue = WorkQueueService(self.config.AnalyticsDataCollector.localQueueURL) # set the connection for local couchDB call self.localCouchDB = LocalCouchDBData(self.config.AnalyticsDataCollector.localCouchURL, self.config.JobStateMachine.summaryStatsDBName, self.summaryLevel) # interface to WMBS/BossAir db myThread = threading.currentThread() # set wmagent db data self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi, myThread.logger) # set the connection for local couchDB call self.localSummaryCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.localWMStatsURL, appName="WMStatsAgent") if hasattr(self.config, "Tier0Feeder"): #use local db for tier0 centralRequestCouchDBURL = self.config.AnalyticsDataCollector.localT0RequestDBURL else: centralRequestCouchDBURL = self.config.AnalyticsDataCollector.centralRequestDBURL self.centralRequestCouchDB = RequestDBWriter(centralRequestCouchDBURL, couchapp = self.config.AnalyticsDataCollector.RequestCouchApp) #TODO: change the config to hold couch url self.localCouchServer = CouchMonitor(self.config.JobStateMachine.couchurl) if self.pluginName != None: pluginFactory = WMFactory("plugins", "WMComponent.AnalyticsDataCollector.Plugins") self.plugin = pluginFactory.loadObject(classname = self.pluginName)
def __call__(self, parameters): """ Handles the event with payload, by sending it to the threadpool. """ myThread = threading.currentThread() #myThread.transaction.begin() #payload = parameters #I think this is correct; it's all that I've got to go on factory = WMFactory("JobMaker", "WMCore.WMSpec.Makers.Interface") createWorkArea = factory.loadObject("CreateWorkArea") jobGroupID = parameters #There should only be one #We should have, in the payload, exactly one jobGroup ID createWorkArea.getNewJobGroup(jobGroupID) createWorkArea.createJobGroupArea() createWorkArea.createWorkArea() #For testing purposes only #DO NOT LEAVE ACTIVE! #createWorkArea.cleanUpAll() #myThread.transaction.commit() return
def __init__(self ): """ The constructor creates separate instances for objects that we do not want to share. Objects that are used as a thread slave Inherit from this to ensure they do not conflict with these methods. If needed the developer can add additional objects as it sees fit in the threadslave class that inherits from this. """ self.args = {} # we also keep a reference to our component (we can # use this for read only things in the argument list). # assign this later self.component = None #a slave is created in its master thread so we can exploit #this to get a reference to its dbfactory object. myThread = threading.currentThread() self.dbFactory = myThread.dbFactory # get the procid from the mainthread msg service # if we use this in testing it might not be there. self.procid = 0 # we can potentially use mapping from messages to handlers # to have one thread handling multiple message types. self.messages = {} # start loading some objects we need in this thread only factory = WMFactory("threadPool", "WMCore.ThreadPool."+ \ myThread.dialect) self.query = factory.loadObject("Queries")
def __init__(self): """ The constructor creates separate instances for objects that we do not want to share. Objects that are used as a thread slave Inherit from this to ensure they do not conflict with these methods. If needed the developer can add additional objects as it sees fit in the threadslave class that inherits from this. """ self.args = {} # we also keep a reference to our component (we can # use this for read only things in the argument list). # assign this later self.component = None #a slave is created in its master thread so we can exploit #this to get a reference to its dbfactory object. myThread = threading.currentThread() self.dbFactory = myThread.dbFactory # get the procid from the mainthread msg service # if we use this in testing it might not be there. self.procid = 0 # we can potentially use mapping from messages to handlers # to have one thread handling multiple message types. self.messages = {} # start loading some objects we need in this thread only factory = WMFactory("threadPool", "WMCore.ThreadPool."+ \ myThread.dialect) self.query = factory.loadObject("Queries")
def setup(self, parameters): """ set db connection(couchdb, wmbs) to prepare to gather information """ # set the connection to local queue self.localQueue = WorkQueueService(self.config.AnalyticsDataCollector.localQueueURL) # set the connection for local couchDB call self.localCouchDB = LocalCouchDBData(self.config.AnalyticsDataCollector.localCouchURL, self.config.JobStateMachine.summaryStatsDBName, self.summaryLevel) # interface to WMBS/BossAir db myThread = threading.currentThread() # set wmagent db data self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi, myThread.logger) # set the connection for local couchDB call self.localSummaryCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.localWMStatsURL, "WMStatsAgent") if hasattr(self.config, "Tier0Feeder"): #use local db for tier0 centralRequestCouchDBURL = self.config.AnalyticsDataCollector.localT0RequestDBURL else: centralRequestCouchDBURL = self.config.AnalyticsDataCollector.centralRequestDBURL self.centralRequestCouchDB = RequestDBWriter(centralRequestCouchDBURL, couchapp = self.config.AnalyticsDataCollector.RequestCouchApp) #TODO: change the config to hold couch url self.localCouchServer = CouchMonitor(self.config.JobStateMachine.couchurl) if self.pluginName != None: pluginFactory = WMFactory("plugins", "WMComponent.AnalyticsDataCollector.Plugins") self.plugin = pluginFactory.loadObject(classname = self.pluginName)
def testF_DBSUploadQueueSizeCheckForAlerts(self): """ Test will not trigger a real alert being sent unless doing some mocking of the methods used during DBSUploadPoller.algorithm() -> DBSUploadPoller.uploadBlocks() method. As done here, it probably can't be deterministic, yet the feature shall be checked. """ sizeLevelToTest = 1 myThread = threading.currentThread() config = self.createConfig() # threshold / value to check config.DBSUpload.alertUploadQueueSize = sizeLevelToTest # without this uploadBlocks method returns immediately name = "ThisIsATest_%s" % (makeUUID()) tier = "RECO" nFiles = sizeLevelToTest + 1 files = self.getFiles(name = name, tier = tier, nFiles = nFiles) datasetPath = '/%s/%s/%s' % (name, name, tier) # load components that are necessary to check status # (this seems necessary, else some previous tests started failing) factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database.Interface") dbinterface = factory.loadObject("UploadToDBS") dbsInterface = DBSInterface(config = config) localAPI = dbsInterface.getAPIRef() globeAPI = dbsInterface.getAPIRef(globalRef = True) testDBSUpload = DBSUploadPoller(config) # this is finally where the action (alert) should be triggered from testDBSUpload.algorithm() return
def initializeSchema(self, nameSpace, modules = []): """ Sometimes you need to initialize the schema before starting the program. This methods lets you pass modules that have an execute method which contains arbitrary sql statements. """ myThread = threading.currentThread() # filter out unique modules myThread.transaction.begin() factory = WMFactory(nameSpace, nameSpace + "." + myThread.dialect) for factoryName in modules: # need to create these tables for testing. # notice the default structure: <dialect>/Create create = factory.loadObject(factoryName) create.logger = logging create.dbi = myThread.dbi createworked = create.execute(conn = myThread.transaction.conn) if createworked: logging.debug("SQL for "+ factoryName + " executed") else: logging.debug("SQL " + factoryName + \ " could not be executed, already exists?") myThread.transaction.commit()
def preInitialization(self): print "DBSBuffer.preInitialization" # use a factory to dynamically load handlers. factory = WMFactory('generic') self.messages['JobSuccess'] = \ factory.loadObject(self.config.DBSBuffer.jobSuccessHandler, self)
def __call__(self, parameters): """ Handles the event with payload, by sending it to the threadpool. """ myThread = threading.currentThread() #myThread.transaction.begin() #payload = parameters #I think this is correct; it's all that I've got to go on factory = WMFactory("JobMaker", "WMCore.WMSpec.Makers.Interface") createWorkArea=factory.loadObject("CreateWorkArea") jobGroupID = parameters #There should only be one #We should have, in the payload, exactly one jobGroup ID createWorkArea.getNewJobGroup(jobGroupID) createWorkArea.createJobGroupArea() createWorkArea.createWorkArea() #For testing purposes only #DO NOT LEAVE ACTIVE! #createWorkArea.cleanUpAll() #myThread.transaction.commit() return
def setSchema(self, modules=None, params=None): """ Creates the schema in the database based on the modules input. This method needs to have been preceded by the setDatabaseConnection. """ modules = modules or [] myThread = threading.currentThread() parameters = None flag = False # Set up for typical DBCreator format: logger, dbi, params if params != None: parameters = [None, None, params] flag = True myThread.transaction.begin() for factoryName in modules: # need to create these tables for testing. # notice the default structure: <dialect>/Create factory = WMFactory(factoryName, factoryName + "." + myThread.dialect) create = factory.loadObject("Create", args=parameters, listFlag=flag) createworked = create.execute(conn=myThread.transaction.conn, transaction=myThread.transaction) if createworked: logging.debug("Tables for " + factoryName + " created") else: logging.debug("Tables " + factoryName + " could not be created.") myThread.transaction.commit()
def get_request_template_from_type(request_type, loc="WMSpec.StdSpecs"): pluginFactory = WMFactory("specArgs", loc) alteredClassName = "%sWorkloadFactory" % request_type spec = pluginFactory.loadObject(classname=request_type, alteredClassName=alteredClassName) specArgs = spec.getWorkloadArguments() result = create_json_template_spec(specArgs) return result
def get_request_template_from_type(request_type, loc="WMSpec.StdSpecs"): pluginFactory = WMFactory("specArgs", loc) alteredClassName = "%sWorkloadFactory" % request_type spec = pluginFactory.loadObject(classname = request_type, alteredClassName = alteredClassName) specArgs = spec.getWorkloadArguments() result = create_json_template_spec(specArgs) return result
class LFNSourceDuplicator(BaseWorkerThread): """ _LFNSourceDuplicator_ Load plugin to get the result of the source database query. Duplicates the result got into the local database. """ def __init__(self, config): BaseWorkerThread.__init__(self) self.config = config.AsyncTransfer # self.logger is set up by the BaseWorkerThread, we just set it's level try: self.logger.setLevel(self.config.log_level) except: import logging self.logger = logging.getLogger() self.logger.setLevel(self.config.log_level) self.logger.debug('Configuration loaded') # Set up a factory for loading plugins self.factory = WMFactory(self.config.pluginDir, namespace = self.config.pluginDir) # Asynch db server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy) self.db = server.connectDatabase(self.config.files_database) self.logger.debug('Connected to CouchDB') return def algorithm(self, parameters = None): """ _algorithm_ Load the plugin of a db source which load its view. Duplicates the results got from the plugin in Async database. """ self.logger.debug('Duplication algorithm begins') try: duplicator = self.factory.loadObject(self.config.pluginName, args = [self.config, self.logger], getFromCache = True, listFlag = True) except ImportError,e : msg = "plugin \'%s\' unknown" % self.config.pluginName self.logger.info(msg) self.logger.info(e) for doc in duplicator(): self.db.queue(doc, True, ['AsyncTransfer/ftscp'] ) self.logger.debug("doc queued %s" %doc) self.db.commit(viewlist=['AsyncTransfer/ftscp']) self.logger.debug('Duplication done') return
def __init__(self, config): self.config = config # Saves the config because Page needs it self.encoder = json.JSONEncoder() # The full URL of this openid server if hasattr(config, 'server_url'): self.base_path = '%s/' % config.server_url else: self.base_path = '%s/' % cherrypy.url() # Now instantiates the OpenID Store (used for nonces and associations) if config.store == 'filestore': store = FileOpenIDStore(config.store_path) elif config.store == 'dbstore': store = DBOidStore(config.store_source) else: store = MemoryStore() # Now instantiates the OpenID server self.openid = server.Server(store, self.base_path + 'openidserver') # Record how to get user details out... factory = WMFactory('oid_factory') self.userstore = factory.loadObject(config.users_store.object, config.users_store) # List of remote sites allowed to request auth/authz to this server self.truststore = factory.loadObject(config.trust_store.object, config.trust_store) # This is a thread safe object to display web pages/forms related to # the oid server. I put it on a separate class to avoid blowing # the main class with html code. self.webpages = OidServerWebPages() # *** Should enable cherrypy session somewhere... sorry cherrypy.config.update({ 'tools.sessions.on': True, 'tools.sessions.storage_type': 'ram', #'tools.sessions.storage_type': "file", #'tools.sessions.storage_path': "/tmp/oidserver-sessions", 'tools.sessions.timeout': 60, 'tools.sessions.name': 'oidserver_sid' })
def getDBinstance(config, namespace, name): if config.backend.lower() == 'mysql': backend = 'MySQL' elif config.backend.lower() == 'oracle': backend = 'Oracle' #factory = WMFactory(name = 'TaskQuery', namespace = 'Databases.TaskDB.%s.Task' % backend) factory = WMFactory(name = name, namespace = 'Databases.%s.%s.%s' % (namespace,backend,name)) return factory.loadObject( name )
def __init__(self, config): self.config = config # Saves the config because Page needs it self.encoder = json.JSONEncoder() # The full URL of this openid server if hasattr(config,'server_url'): self.base_path = '%s/' % config.server_url else: self.base_path = '%s/' % cherrypy.url() # Now instantiates the OpenID Store (used for nonces and associations) if config.store == 'filestore': store = FileOpenIDStore(config.store_path) elif config.store == 'dbstore': store = DBOidStore(config.store_source) else: store = MemoryStore() # Now instantiates the OpenID server self.openid = server.Server(store, self.base_path + 'openidserver') # Record how to get user details out... factory = WMFactory('oid_factory') self.userstore = factory.loadObject(config.users_store.object, config.users_store) # List of remote sites allowed to request auth/authz to this server self.truststore = factory.loadObject(config.trust_store.object, config.trust_store) # This is a thread safe object to display web pages/forms related to # the oid server. I put it on a separate class to avoid blowing # the main class with html code. self.webpages = OidServerWebPages() # *** Should enable cherrypy session somewhere... sorry cherrypy.config.update({'tools.sessions.on': True, 'tools.sessions.storage_type': 'ram', #'tools.sessions.storage_type': "file", #'tools.sessions.storage_path': "/tmp/oidserver-sessions", 'tools.sessions.timeout': 60, 'tools.sessions.name': 'oidserver_sid'})
def updateDSFileCount(self, dataset): myThread = threading.currentThread() existingTransaction = self.beginTransaction() factory = WMFactory("dbsBuffer", "WMComponent.DBSBuffer.Database."+ \ myThread.dialect) newDS = factory.loadObject("UpdateDSFileCount") newDS.execute(dataset=dataset, conn = self.getDBConn(), transaction=self.existingTransaction()) self.commitTransaction(existingTransaction) return
def __init__(self, params = None): """ Constructor. The 'params' argument can be used as a dict for any parameter (if needed). Basic things can be obtained from currentThread. The required params are as follow: downloadBaseUrl, sandboxBasePath, specBasePath, matcherPlugin """ myThread = threading.currentThread() # factory = WMFactory("default", \ # "TQComp.Database."+myThread.dialect) factory = WMFactory("default") # self.queries = factory.loadObject("Queries") self.queries = factory.loadObject("TQComp.Database."\ +myThread.dialect+".Queries") self.logger = myThread.logger required = ["downloadBaseUrl", "sandboxBasePath", "specBasePath",\ "uploadBaseUrl", "matcherPlugin", "maxThreads"] for param in required: if not param in params: messg = "GetTaskHandler object requires params['%s']" % param # TODO: What number? numb = 0 raise WMException(messg, numb) self.downloadBaseUrl = params["downloadBaseUrl"] self.sandboxBasePath = params["sandboxBasePath"] self.specBasePath = params["specBasePath"] self.uploadBaseUrl = params["uploadBaseUrl"] self.maxThreads = params["maxThreads"] try: self.matcher = factory.loadObject(params["matcherPlugin"],\ {'queries': self.queries, 'logger': self.logger}) except: ttype, val, tb = sys.exc_info() # myThread.transaction.rollback() messg = 'Could not load matcher plugin, due to: %s - %s '% (ttype, val) # TODO: What number? numb = 0 raise WMException(messg, numb)
class RetryManagerPoller(BaseWorkerThread): """ _RetryManagerPoller_ Polls for Jobs in CoolOff State and attempts to retry them based on the requirements in the selected plugin """ def __init__(self, config): """ Initialise class members """ BaseWorkerThread.__init__(self) self.config = config myThread = threading.currentThread() self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) pluginPath = getattr(self.config.RetryManager, "pluginPath", "WMComponent.RetryManager.PlugIns") self.pluginFactory = WMFactory("plugins", pluginPath) self.changeState = ChangeState(self.config) self.getJobs = self.daoFactory(classname="Jobs.GetAllJobs") # initialize the alert framework (if available) (self.sendAlert()) self.initAlerts(compName="RetryManager") # get needed plugins self.plugins = {} self.typePluginsAssoc = getattr(self.config.RetryManager, 'plugins', {}) self.typePluginsAssoc.setdefault('default', 'DefaultRetryAlgo') for pluginName in self.typePluginsAssoc.values(): try: plugin = self.pluginFactory.loadObject(classname=pluginName, args=config) self.plugins[pluginName] = plugin except Exception, ex: msg = "Error loading plugin %s on path %s\n" % (pluginName, pluginPath) msg += str(ex) logging.error(msg) self.sendAlert(6, msg=msg) raise RetryManagerException(msg) return
class RetryManagerPoller(BaseWorkerThread): """ _RetryManagerPoller_ Polls for Jobs in CoolOff State and attempts to retry them based on the requirements in the selected plugin """ def __init__(self, config): """ Initialise class members """ BaseWorkerThread.__init__(self) self.config = config myThread = threading.currentThread() self.daoFactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) pluginPath = getattr(self.config.RetryManager, "pluginPath", "WMComponent.RetryManager.PlugIns") self.pluginFactory = WMFactory("plugins", pluginPath) self.changeState = ChangeState(self.config) self.getJobs = self.daoFactory(classname = "Jobs.GetAllJobs") # initialize the alert framework (if available) (self.sendAlert()) self.initAlerts(compName = "RetryManager") # get needed plugins self.plugins = {} self.typePluginsAssoc = getattr(self.config.RetryManager, 'plugins', {}) self.typePluginsAssoc.setdefault('default', 'DefaultRetryAlgo') for pluginName in self.typePluginsAssoc.values(): try: plugin = self.pluginFactory.loadObject(classname = pluginName, args = config) self.plugins[pluginName] = plugin except Exception, ex: msg = "Error loading plugin %s on path %s\n" % (pluginName, pluginPath) msg += str(ex) logging.error(msg) self.sendAlert(6, msg = msg) raise RetryManagerException(msg) return
class GeneratorFactory(WMFactory): """ _GeneratorFactory_ Instantiate a WMFactory instance with the appropriate namespace """ def __init__(self): self.factory = WMFactory(self.__class__.__name__, "WMCore.JobSplitting.Generators") def getGenerator(self, generatorName, **options): """ _getGenerator_ factory method to return a step template instance based on the name of the step """ args = {} args.update(options) try: return self.factory.loadObject(generatorName, args) except WMException as wmEx: msg = "GeneratorFactory Unable to load Object: %s" % generatorName raise GeneratorFactoryException(msg) except Exception as ex: msg = "Error creating object %s in GeneratorFactory:\n" % generatorName msg += str(ex) raise GeneratorFactoryException(msg) return def makeGenerators(self, wmTaskHelper): """ _makeGenerators_ Util function to build the set of generators defined by the task provided """ result = [] for generator in wmTaskHelper.listGenerators(): genInstance = self.getGenerator(generator, wmTaskHelper, **wmTaskHelper.getGeneratorSettings(generator) ) result.append(genInstance) return result
def tearDown(self): """ _tearDown_ Drop all the WMBS tables. """ myThread = threading.currentThread() factory = WMFactory("WMBS", "WMCore.WMBS") destroy = factory.loadObject(myThread.dialect + ".Destroy") myThread.transaction.begin() destroyworked = destroy.execute(conn=myThread.transaction.conn) if not destroyworked: raise Exception("Could not complete WMBS tear down.") myThread.transaction.commit()
def setBlockStatus(self, block, locations, openStatus=0, time=0): """ _setBlockStatus_ Adds information about the block from DBSWriter to the database """ myThread = threading.currentThread() existingTransaction = self.beginTransaction() factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database."+ \ myThread.dialect) newDS = factory.loadObject("SetBlockStatus") newDS.execute(block = block, locations = locations, open_status = openStatus, time = time, \ conn = self.getDBConn(), transaction=self.existingTransaction()) self.commitTransaction(existingTransaction) return
def setBlockStatus(self, block, locations, openStatus = 0, time = 0): """ _setBlockStatus_ Adds information about the block from DBSWriter to the database """ myThread = threading.currentThread() existingTransaction = self.beginTransaction() factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database."+ \ myThread.dialect) newDS = factory.loadObject("SetBlockStatus") newDS.execute(block = block, locations = locations, open_status = openStatus, time = time, \ conn = self.getDBConn(), transaction=self.existingTransaction()) self.commitTransaction(existingTransaction) return
class GeneratorFactory(WMFactory): """ _GeneratorFactory_ Instantiate a WMFactory instance with the appropriate namespace """ def __init__(self): self.factory = WMFactory(self.__class__.__name__, "WMCore.JobSplitting.Generators") def getGenerator(self, generatorName, **options): """ _getGenerator_ factory method to return a step template instance based on the name of the step """ args = {} args.update(options) try: return self.factory.loadObject(generatorName, args) except WMException as wmEx: msg = "GeneratorFactory Unable to load Object: %s" % generatorName raise GeneratorFactoryException(msg) except Exception as ex: msg = "Error creating object %s in GeneratorFactory:\n" % generatorName msg += str(ex) raise GeneratorFactoryException(msg) return def makeGenerators(self, wmTaskHelper): """ _makeGenerators_ Util function to build the set of generators defined by the task provided """ result = [] for generator in wmTaskHelper.listGenerators(): genInstance = self.getGenerator( generator, wmTaskHelper, **wmTaskHelper.getGeneratorSettings(generator)) result.append(genInstance) return result
def initInThread(self): """ Called during thread initialization. Loads the backend for this instance. """ # make sure you instantiate the super class method. ThreadSlave.initInThread(self) # load queries for backend. myThread = threading.currentThread() factory = WMFactory("default", \ "TQComp.Database."+myThread.dialect) # make sure you do not overload attributes # defined in your parent class self.queries = factory.loadObject("Queries") logging.debug("ParentSlave initialized")
def preInitialization(self): if not hasattr(self.config.PilotMonitorComponent, "pilotMonitorHandler"): self.config.PilotMonitorComponent.pilotMonitorHandler = \ 'PilotMonitor.Handler.PilotMonitorHandler' factory = WMFactory("PilotMonitor"); self.messages["MonitorPilots"] = factory.loadObject(\ self.config.PilotMonitorComponent.pilotMonitorHandler, self) tqconfig = Configuration.loadConfigurationFile( \ self.config.PilotMonitorComponent.TQConfig ) myThread = threading.currentThread() self.logger = myThread.logger self.logger.debug( tqconfig) self.tqStateApi = TQStateApi(self.logger, tqconfig, None)
def clearDatabase(self, modules = []): """ Enables clearing particular tables in the database Associated to modules. Note this only works if there is the module has a Destroy class. Beware that this might not work if there are table dependencies. """ myThread = threading.currentThread() for module in modules: factory = WMFactory("clear", module) destroy = factory.loadObject(myThread.dialect+".Destroy") myThread.transaction.begin() destroyworked = destroy.execute(conn = myThread.transaction.conn) if not destroyworked: raise Exception(module +" tables could not be destroyed") myThread.transaction.commit() del factory
def findAlgos(self, dataset): """ Call to findAlgos """ myThread = threading.currentThread() existingTransaction = self.beginTransaction() factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database."+ \ myThread.dialect) findAlgos = factory.loadObject("FindAlgos") # Add the file to the buffer (API Call) #results = findFiles.execute(conn = self.getDBConn(), transaction=self.existingTransaction()) results = findAlgos.execute(datasetInfo=dataset, conn = self.getDBConn(), transaction=self.existingTransaction()) self.commitTransaction(existingTransaction) return results
def testE_NoMigration(self): """ _NoMigration_ Test the DBSUpload system with no global migration """ myThread = threading.currentThread() config = self.createConfig() self.injectWorkflow(MaxWaitTime=3) config.DBSInterface.doGlobalMigration = False config.DBSUpload.pollInterval = 4 name = "ThisIsATest_%s" % (makeUUID()) tier = "RECO" nFiles = 12 files = self.getFiles(name=name, tier=tier, nFiles=nFiles) datasetPath = '/%s/%s/%s' % (name, name, tier) # Load components that are necessary to check status factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database.Interface") dbinterface = factory.loadObject("UploadToDBS") dbsInterface = DBSInterface(config=config) localAPI = dbsInterface.getAPIRef() globeAPI = dbsInterface.getAPIRef(globalRef=True) # In the first round we should create blocks for the first dataset # The child dataset should not be handled until the parent is uploaded testDBSUpload = DBSUploadPoller(config=config) testDBSUpload.algorithm() # First, see if there are any blocks # One in DBS, one not in DBS result = myThread.dbi.processData( "SELECT status FROM dbsbuffer_block")[0].fetchall() self.assertEqual(len(result), 2) self.assertEqual(result, [('InGlobalDBS', ), ('Open', )]) result = myThread.dbi.processData( "SELECT status FROM dbsbuffer_file WHERE dataset_algo = 1" )[0].fetchall() for r in result: self.assertEqual(r[0], 'GLOBAL') return
def runDas(self, func, data, expires): """ Run a query and produce a dictionary for DAS formatting """ start_time = time.time() results = func(self, data) call_time = time.time() - start_time res_expire = make_timestamp(expires) if isinstance(results, list): if len(results) > 0: row = results[0] else: row = None else: row = results if isinstance(row, str): row = '"%s"' % row try: factory = WMFactory('webtools_factory') obj = factory.loadObject(self.config.model.object, self.config) res_version = obj.version except Exception: res_version = 'unknown' keyhash = hashlib.md5() if not isinstance(results, (str, bytes)): keyhash.update(encodeUnicodeToBytesConditional(str(results), condition=PY3)) else: keyhash.update(encodeUnicodeToBytesConditional(results, condition=PY3)) res_checksum = keyhash.hexdigest() dasdata = {'application': '%s.%s' % (self.config.application, func.__name__), 'request_timestamp': start_time, 'request_url': request.base + request.path_info + '?' + request.query_string, 'request_method': request.method, 'request_params': request.params, 'response_version': res_version, 'response_expires': res_expire, 'response_checksum': res_checksum, 'request_call': func.__name__, 'call_time': call_time, 'results': results, } return dasdata
def findAlgos(self, dataset): """ Call to findAlgos """ myThread = threading.currentThread() existingTransaction = self.beginTransaction() factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database."+ \ myThread.dialect) findAlgos = factory.loadObject("FindAlgos") # Add the file to the buffer (API Call) #results = findFiles.execute(conn = self.getDBConn(), transaction=self.existingTransaction()) results = findAlgos.execute(datasetInfo=dataset, conn=self.getDBConn(), transaction=self.existingTransaction()) self.commitTransaction(existingTransaction) return results
def testD(self): raise nose.SkipTest config = self.testInit.getConfiguration() config.component_("TestComponent") config.TestComponent.logLevel = 'INFO' config.section_("General") self.tempDir = self.testInit.generateWorkDir(config) # try starting a component as a daemon: config.TestComponent.componentDir = os.path.join( \ self.tempDir, "Components/TestComponent2") os.makedirs(config.TestComponent.componentDir) testComponent = TestComponent(config) # we set the parent to true as we are testing testComponent.startDaemon(keepParent=True) time.sleep(2) daemonFile = os.path.join(config.TestComponent.componentDir, "Daemon.xml") details = Details(daemonFile) print('Is component alive: ' + str(details.isAlive())) # create msgService to send stop message. myThread = threading.currentThread() factory = WMFactory("msgService", "WMCore.MsgService." + \ myThread.dialect) myThread.transaction = Transaction(myThread.dbi) msgService = factory.loadObject("MsgService") msgService.registerAs("HarnessTest") myThread.transaction.commit() print( 'Publish a stop message to test if the component shutsdown gracefully' ) myThread.transaction.begin() msg = {'name': 'Stop', 'payload': ''} msgService.publish(msg) myThread.transaction.commit() msgService.finish() while details.isAlive(): print('Component has not received stop message') time.sleep(2) print('Daemon shutdown gracefully')
def runDas(self, func, data, expires): """ Run a query and produce a dictionary for DAS formatting """ start_time = time.time() results = func(self, data) call_time = time.time() - start_time res_expire = make_timestamp(expires) if type(results) is types.ListType: if len(results) > 0: row = results[0] else: row = None else: row = results if type(row) is types.StringType: row = '"%s"' % row try: factory = WMFactory('webtools_factory') object = factory.loadObject(self.config.model.object, self.config) res_version = object.version except: res_version = 'unknown' keyhash = hashlib.md5() keyhash.update(str(results)) res_checksum = keyhash.hexdigest() dasdata = {'application':'%s.%s' % (self.config.application, func.__name__), 'request_timestamp': start_time, 'request_url': request.base + request.path_info + '?' + \ request.query_string, 'request_method' : request.method, 'request_params' : request.params, 'response_version': res_version, 'response_expires': res_expire, 'response_checksum': res_checksum, 'request_call': func.__name__, 'call_time': call_time, 'results': results, } return dasdata
def setup(self, parameters): """ set db connection(couchdb, wmbs) to prepare to gather information """ # set the connection to local queue self.localQueue = WorkQueueService(self.config.AnalyticsDataCollector.localQueueURL) # set the connection for local couchDB call self.localCouchDB = LocalCouchDBData(self.config.AnalyticsDataCollector.localCouchURL, self.summaryLevel) # interface to WMBS/BossAir db myThread = threading.currentThread() # set wmagent db data self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi, myThread.logger) # set the connection for local couchDB call self.localSummaryCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.localWMStatsURL) self.centralWMStatsCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.centralWMStatsURL) if self.pluginName != None: pluginFactory = WMFactory("plugins", "WMComponent.AnalyticsDataCollector.Plugins") self.plugin = pluginFactory.loadObject(classname = self.pluginName)
def __init__(self, config): myThread = threading.currentThread() self.config = config self.dbsurl = self.config.DBSUpload.dbsurl self.dbsversion = self.config.DBSUpload.dbsversion self.uploadFileMax = self.config.DBSUpload.uploadFileMax self.DBSMaxFiles = self.config.DBSUpload.DBSMaxFiles self.DBSMaxSize = self.config.DBSUpload.DBSMaxSize self.DBSBlockTimeout = self.config.DBSUpload.DBSBlockTimeout self.dbswriter = DBSWriter(self.dbsurl, level='ERROR', user='******', version=self.dbsversion, \ globalDBSUrl = self.config.DBSUpload.globalDBSUrl, \ globalVersion = self.config.DBSUpload.globalDBSVer) self.daoFactory = DAOFactory(package="WMComponent.DBSBuffer.Database", logger=logging, dbinterface=myThread.dbi) factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database.Interface") self.dbinterface = factory.loadObject("UploadToDBS")
def preInitialization(self): """ Add required worker modules to work threads """ # in case nothing was configured we have a fallback. if not hasattr(self.config.FeederManager, "addDatasetWatch"): logging.warning("Using default AddDatasetWatch handler") self.config.FeederManager.addDatasetWatchHandler = \ 'WMComponent.FeederManager.Handler.DefaultAddDatasetWatch' # use a factory to dynamically load handlers. factory = WMFactory('generic') self.messages['AddDatasetWatch'] = \ factory.loadObject(\ self.config.FeederManager.addDatasetWatchHandler, self) myThread = threading.currentThread() myThread.runningFeedersLock = threading.Lock() myThread.runningFeeders = {} pollInterval = self.config.FeederManager.pollInterval logging.info("Setting poll interval to %s seconds" % pollInterval) myThread.workerThreadManager.addWorker(FeederManagerPoller(), \ pollInterval)
class Watchdog(threading.Thread): """ Watchdog It's like a watchdog! It watches things, and then it barks at them, curls up, and goes to sleep Or possibly monitors them and records what they do. It's a bit of a crapshoot at this point. """ def __init__(self, logPath=None, config=None): threading.Thread.__init__(self) self.doMonitoring = True self._Finished = threading.Event() self._EndOfJob = threading.Event() self._NewTask = threading.Event() self._JobKilled = threading.Event() self._RunUpdate = threading.Event() self._Interval = 120.0 self._Monitors = [] # Right now we join this, because we don't know # Where we'll be when we need this. self.logPath = os.path.join(os.getcwd(), logPath) self.factory = WMFactory(self.__class__.__name__, "WMCore.WMRuntime.Monitors") def setupMonitors(self, task, wmbsJob): logging.info("In Watchdog.setupMonitors") if not hasattr(task.data, 'watchdog'): msg = "Could not find watchdog in spec" logging.error(msg) # I don't think this is necessarily fatal return if not hasattr(task.data.watchdog, 'monitors'): msg = "Watchdog has no monitors" logging.error(msg) # Probably not fatal either return if hasattr(task.data.watchdog, 'interval'): # Set the interval off the config self.setInterval(task.data.watchdog.interval) for monitor in task.data.watchdog.monitors: msg = "Initializing monitor %s" % monitor logging.info(msg) mon = self.loadMonitor(monitor) args = {} if hasattr(task.data.watchdog, monitor): # This should be a config section monitorArgs = getattr(task.data.watchdog, monitor) args = monitorArgs.dictionary_() if monitor == 'PerformanceMonitor' and args: # Apply tweaks to PerformanceMonitor only. # Scale resources according to the HTCondor runtime environment. origCores = 1 for stepName in task.listAllStepNames(): sh = task.getStepHelper(stepName) origCores = max(origCores, sh.getNumberOfCores()) resources = {'cores': origCores} origMaxPSS = args.get('maxPSS', args.get('maxRSS')) if origMaxPSS: resources['memory'] = origMaxPSS # Actually parses the HTCondor runtime resizeResources(resources) # We decided to only touch Watchdog settings if the number of cores changed. # (even if this means the watchdog memory is wrong for a slot this size). changedCores = origCores != resources['cores'] # If we did base maxPSS off the memory in the HTCondor slot, subtract a bit # off the top so watchdog triggers before HTCondor does. # Add the new number of cores to the args such that PerformanceMonitor can see it args['cores'] = resources['cores'] if changedCores: if origMaxPSS: args['maxPSS'] = resources['memory'] - 50 logging.info("Watchdog modified: %s. Final settings:", changedCores) for k, v in viewitems(args): logging.info(" %s: %r", k, v) # Actually initialize the monitor variables mon.initMonitor(task=task, job=wmbsJob, logPath=self.logPath, args=args) self._Monitors.append(mon) return def loadMonitor(self, monitorName): try: return self.factory.loadObject(monitorName) except WMException: msg = "WatchdogFactory Unable to load Object: %s" % monitorName logging.error(msg) raise WatchdogException(msg) except Exception as ex: msg = "Error creating object %s in WatchdogFactory:\n" % monitorName msg += str(ex) logging.error(msg) raise WatchdogException(msg) def setInterval(self, interval): """ Set the monitor interval """ logging.info("Set Watchdog interval to %s", interval) self._Interval = interval def disableMonitoring(self): """ _disableMonitoring_ Turn off active monitoring (periodicUpdate calls are disabled, event driven monitoring still occurs """ self.doMonitoring = False def shutdown(self): """ Shutdown the monitor. """ logging.info("MonitorState: Shutdown called") self._Finished.set() return # //=========notify Methods called by the ExecutionManager==== # // # // Start notification from the exe thread, this starts the # //periodic updates of the monitor thread # // # // def notifyJobStart(self, task): """ Start the job. """ logging.info("MonitorThread: JobStarted") if self.doMonitoring: self.setDaemon(1) self.start() for monitor in self._Monitors: try: monitor.jobStart(task) except Exception as ex: msg = "Error in notifyJobStart for monitor class %s in Watchdog:\n" % monitor.__class__ msg += str(ex) msg += str(traceback.format_exc()) logging.error(msg) raise WatchdogException(msg) return # // # // notify Monitors of new task start up # // def notifyStepStart(self, step): """ notify Monitors of new task start up. """ self._RunUpdate.set() for monitor in self._Monitors: try: monitor.stepStart(step) except Exception as ex: msg = "Error in notifyTaskStart for monitor class %s in Watchdog:\n" % monitor.__class__ msg += str(ex) msg += str(traceback.format_exc()) logging.error(msg) raise WatchdogException(msg) return # // # // notify Monitors of task completion # // def notifyStepEnd(self, step, exitCode=0, stepReport=None): """ notify Monitors of task completion. """ self._RunUpdate.clear() for monitor in self._Monitors: try: monitor.stepEnd(step=step, stepReport=stepReport) except Exception as ex: msg = "Error in notifyTaskEnd for monitor class %s in Watchdog:\n" % monitor.__class__ msg += str(ex) msg += str(traceback.format_exc()) logging.error(msg) raise WatchdogException(msg) # print "Task Ended: %s with Exit Code:%s" % (task, exitCode) # self._MonMgr.taskEnd(task, exitCode) return # // # // notify monitors of Job Completion, stops the periodic # // updating def notifyJobEnd(self, task): """ notify monitors of Job Completion, stops the periodic updating. """ logging.info("MonitorThread: JobEnded") for monitor in self._Monitors: try: monitor.jobEnd(task) except Exception as ex: msg = "Error in notifyJobEnd for monitor class %s in Watchdog:\n" % monitor.__class__ msg += str(ex) msg += str(traceback.format_exc()) raise WatchdogException(msg) # self._MonMgr.jobEnd() self.shutdown() return # // # // Interrupt Notifiers # // Job has been killed def notifyKillJob(self): """ Interrupt Notifiers, Job has been killed. """ logging.info("MonitorThread: JobKilled") for monitor in self._Monitors: try: monitor.jobKilled() except Exception as ex: msg = "Error in notifyKillJob for monitor class %s in Watchdog:\n" % monitor.__class__ msg += str(ex) msg += str(traceback.format_exc()) logging.error(msg) raise WatchdogException(msg) # self._MonMgr.jobKilled() self.shutdown() # // # // Task has been killed # // def notifyKillStep(self, step=None): """ Task has been killed. """ logging.info("MonitorThread: TaskKilled") for monitor in self._Monitors: try: monitor.stepKilled(step) except Exception as ex: msg = "Error in notifyKillTask for monitor class %s in Watchdog:\n" % monitor.__class__ msg += str(ex) msg += str(traceback.format_exc()) logging.error(msg) raise WatchdogException(msg) # self._MonMgr.taskKilled() # // # // Override Thread.run() to do the periodic update # // of the MonitorState object and dispatch it to the monitors def run(self): """ Override Thread.run() to do the periodic update of the MonitorState object and dispatch it to the monitors """ while True: # // # // shutdown signal # // if self._Finished.isSet(): return # // # // Update State information only during a running task # // if self._RunUpdate.isSet(): for monitor in self._Monitors: try: monitor.periodicUpdate() except Exception as ex: msg = "Error in periodicUpdate for monitor class %s in Watchdog:\n" % monitor.__class__ msg += str(ex) msg += str(traceback.format_exc()) msg += "This is a CRITICAL error because this kills the monitoring.\n" msg += "Terminate thread and retry.\n" logging.error(msg) # raise WatchdogException(msg) # This one needs to be killed by itself # since it's run by thread os.abort() # self._MonMgr.periodicUpdate() # time.sleep(self._Interval) self._Finished.wait(self._Interval) # // # // Load Monitor Objects based on Cfg settings passed # // from Executor def initMonitorFwk(self, monitorCfg, updatorCfg): """ _initMonitorFwk_ Initialise the MonitorMgr object when this method is called from the Execution thread Load Monitor Objects based on Cfg settings passed from Executor. """ # self._MonMgr.monitorConfig = monitorCfg # self._MonMgr.updatorConfig = updatorCfg # self._MonMgr.loadMonitors() # self._MonMgr.loadUpdators() return
def testBulkLoad(self): """ _testBulkLoad_ Can we load in bulk? """ bufferFactory = WMFactory("dbsBuffer", "WMComponent.DBSBuffer.Database.Interface") addToBuffer = bufferFactory.loadObject("AddToBuffer") bulkLoad = self.daoFactory(classname = "DBSBufferFiles.LoadBulkFilesByID") testFileChildA = DBSBufferFile(lfn = "/this/is/a/child/lfnA", size = 1024, events = 20) testFileChildA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileChildA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileChildB = DBSBufferFile(lfn = "/this/is/a/child/lfnB", size = 1024, events = 20) testFileChildB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileChildB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileChildC = DBSBufferFile(lfn = "/this/is/a/child/lfnC", size = 1024, events = 20) testFileChildC.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileChildC.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFileChildA.create() testFileChildB.create() testFileChildC.create() testFileChildA.setLocation(["se1.fnal.gov", "se1.cern.ch"]) testFileChildB.setLocation(["se1.fnal.gov", "se1.cern.ch"]) testFileChildC.setLocation(["se1.fnal.gov", "se1.cern.ch"]) runSet = set() runSet.add(Run( 1, *[45])) runSet.add(Run( 2, *[67, 68])) testFileChildA.addRunSet(runSet) testFileChildB.addRunSet(runSet) testFileChildC.addRunSet(runSet) testFileChildA.save() testFileChildB.save() testFileChildC.save() setCksumAction = self.daoFactory(classname = "DBSBufferFiles.AddChecksumByLFN") binds = [{'lfn': "/this/is/a/child/lfnA", 'cktype': 'adler32', 'cksum': 201}, {'lfn': "/this/is/a/child/lfnA", 'cktype': 'cksum', 'cksum': 101}, {'lfn': "/this/is/a/child/lfnB", 'cktype': 'adler32', 'cksum': 201}, {'lfn': "/this/is/a/child/lfnB", 'cktype': 'cksum', 'cksum': 101}, {'lfn': "/this/is/a/child/lfnC", 'cktype': 'adler32', 'cksum': 201}, {'lfn': "/this/is/a/child/lfnC", 'cktype': 'cksum', 'cksum': 101}] setCksumAction.execute(bulkList = binds) testFile = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024, events = 10) testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO") testFile.create() testFileChildA.addParents([testFile["lfn"]]) testFileChildB.addParents([testFile["lfn"]]) testFileChildC.addParents([testFile["lfn"]]) binds = [{'id': testFileChildA.exists()}, {'id': testFileChildB.exists()}, {'id': testFileChildC.exists()}] listOfFiles = addToBuffer.loadDBSBufferFilesBulk(fileObjs = binds) #print listOfFiles compareList = ['locations', 'psetHash', 'configContent', 'appName', 'appVer', 'appFam', 'events', 'datasetPath', 'runs'] for f in listOfFiles: self.assertTrue(f['lfn'] in ["/this/is/a/child/lfnA", "/this/is/a/child/lfnB", "/this/is/a/child/lfnC"], "Unknown file in loaded results") self.assertEqual(f['checksums'], {'adler32': '201', 'cksum': '101'}) for parent in f['parents']: self.assertEqual(parent['lfn'], testFile['lfn']) for key in compareList: self.assertEqual(f[key], testFileChildA[key])
def testRegistry(self): logging.debug("Initialize registry") # somewhere (e.g. during initialization) you create a registry for queries registry = WMFactory('my_test_registry', WMFactoryTest._repository) testRegisteredFile = \ registry.loadObject('TestCollection_1.TestRegistryFile1') parameters = {'par1':'val1', 'par2':'val2'} result = testRegisteredFile.doSomething(parameters) self.assertEqual(parameters, result) # this time from cache testRegisteredFile = \ registry.loadObject('TestCollection_1.TestRegistryFile2') parameters = {'par3':'val3', 'par4':'val4'} result = testRegisteredFile.doSomething(parameters) self.assertEqual(parameters, result) testRegisteredFile = \ registry.loadObject('TestCollection_2.TestRegistryFile1') parameters = {'par5':'val5', 'par6':'val6'} result = testRegisteredFile.doSomething(parameters) self.assertEqual(parameters, result) testRegisteredFile = \ registry.loadObject('TestCollection_2.TestRegistryFile2') parameters = {'par7':'val7', 'par8':'val8'} result = testRegisteredFile.doSomething(parameters) self.assertEqual(parameters, result) logging.debug("testing cache") testRegisteredFile = \ registry.loadObject('TestCollection_2.TestRegistryFile2') testRegisteredFile = \ registry.loadObject('TestCollection_2.TestRegistryFile2') testRegisteredFile = \ registry.loadObject('TestCollection_2.TestRegistryFile2') logging.debug("testing multiple instances (no cache)") testRegisteredFileInst1 = \ registry.loadObject('TestCollection_2.TestRegistryFile2', \ storeInCache = False) testRegisteredFileInst2 = \ registry.loadObject('TestCollection_2.TestRegistryFile2', \ storeInCache = False) testRegisteredFileInst3 = \ registry.loadObject('TestCollection_2.TestRegistryFile2', \ storeInCache = False) parameters = {'par9':'val9', 'par10':'val10'} result = testRegisteredFileInst1.doSomething(parameters) self.assertEqual(parameters, result) parameters = {'par11':'val11', 'par12':'val12'} result = testRegisteredFileInst2.doSomething(parameters) self.assertEqual(parameters, result) parameters = {'par13':'val13', 'par14':'val14'} result = testRegisteredFileInst3.doSomething(parameters) self.assertEqual(parameters, result) logging.debug("Retrieving registry from thread attribute and load objects") logging.debug("this way all objects in this thread can use the registry") myThread = threading.currentThread() registry = myThread.factory['my_test_registry'] parameters = {'par15':'val15', 'par16':'val16'} testRegisteredFile = \ registry.loadObject('TestCollection_3.TestRegistryFile1', \ args = parameters) result = testRegisteredFile.doSomething(parameters) self.assertEqual(parameters, result)