class TestInit: """ A set of initialization steps used in many tests. Test can call the methods from this class to initialize their default environment so to minimize code duplication. """ def __init__(self, testClassName="Unknown Class"): self.testClassName = testClassName self.testDir = None self.currModules = [] global hasDatabase self.hasDatabase = hasDatabase if self.hasDatabase: self.init = WMInit() self.deleteTmp = True def __del__(self): if self.deleteTmp: self.delWorkDir() self.attemptToCloseDBConnections() def delWorkDir(self): if (self.testDir != None): try: shutil.rmtree(self.testDir) except: # meh, if it fails, I guess something weird happened pass def setLogging(self, logLevel=logging.INFO): """ Sets logging parameters """ # remove old logging instances. return logger1 = logging.getLogger() logger2 = logging.getLogger(self.testClassName) for logger in [logger1, logger2]: for handler in logger.handlers: handler.close() logger.removeHandler(handler) self.init.setLogging(self.testClassName, self.testClassName, logExists=False, logLevel=logLevel) def generateWorkDir(self, config=None, deleteOnDestruction=True, setTmpDir=False): self.testDir = tempfile.mkdtemp() if config: config.section_("General") config.General.workDir = self.testDir os.environ['TESTDIR'] = self.testDir if os.getenv('WMCORE_KEEP_DIRECTORIES', False): deleteOnDestruction = True logging.info("Generated testDir - %s" % self.testDir) if setTmpDir: os.environ['TMPDIR'] = self.testDir self.deleteTmp = deleteOnDestruction return self.testDir def getBackendFromDbURL(self, dburl): dialectPart = dburl.split(":")[0] if dialectPart == 'mysql': return 'MySQL' elif dialectPart == 'sqlite': return 'SQLite' elif dialectPart == 'oracle': return 'Oracle' elif dialectPart == 'http': return 'CouchDB' else: raise RuntimeError, "Unrecognized dialect %s" % dialectPart def setDatabaseConnection(self, connectUrl=None, socket=None, destroyAllDatabase=False): """ Set up the database connection by retrieving the environment parameters. The destroyAllDatabase option is for testing ONLY. Never flip that switch on in any other instance where you don't know what you're doing. """ if not self.hasDatabase: return config = self.getConfiguration(connectUrl=connectUrl, socket=socket) self.coreConfig = config self.init.setDatabaseConnection(config.CoreDatabase.connectUrl, config.CoreDatabase.dialect, config.CoreDatabase.socket) if trashDatabases or destroyAllDatabase: self.clearDatabase() # Have to check whether or not database is empty # If the database is not empty when we go to set the schema, abort! result = self.init.checkDatabaseContents() if len(result) > 0: msg = "Database not empty, cannot set schema !\n" msg += str(result) logging.error(msg) raise TestInitException(msg) return def setSchema(self, customModules=[], useDefault=True, params=None): """ Creates the schema in the database for the default tables/services: trigger, message service, threadpool. Developers can add their own modules to it using the array customModules which should follow the proper naming convention. if useDefault is set to False, it will not instantiate the schemas in the defaultModules array. """ if not self.hasDatabase: return defaultModules = ["WMCore.WMBS"] if not useDefault: defaultModules = [] # filter out unique modules modules = {} for module in (defaultModules + customModules): modules[module] = 'done' try: self.init.setSchema(modules.keys(), params=params) except Exception, ex: print traceback.format_exc() raise ex # store the list of modules we've added to the DB modules = {} for module in (defaultModules + customModules + self.currModules): modules[module] = 'done' self.currModules = modules.keys() return
class TestInit(object): """ A set of initialization steps used in many tests. Test can call the methods from this class to initialize their default environment so to minimize code duplication. """ def __init__(self, testClassName = "Unknown Class"): self.testClassName = testClassName self.testDir = None self.currModules = [] global hasDatabase self.hasDatabase = hasDatabase if self.hasDatabase: self.init = WMInit() self.deleteTmp = True def __del__(self): if self.deleteTmp: self.delWorkDir() self.attemptToCloseDBConnections() def delWorkDir(self): if self.testDir != None: try: shutil.rmtree( self.testDir ) except: # meh, if it fails, I guess something weird happened pass def setLogging(self, logLevel = logging.INFO): """ Sets logging parameters """ # remove old logging instances. return logger1 = logging.getLogger() logger2 = logging.getLogger(self.testClassName) for logger in [logger1, logger2]: for handler in logger.handlers: handler.close() logger.removeHandler(handler) self.init.setLogging(self.testClassName, self.testClassName, logExists = False, logLevel = logLevel) def generateWorkDir(self, config = None, deleteOnDestruction = True, setTmpDir = False): self.testDir = tempfile.mkdtemp() if config: config.section_("General") config.General.workDir = self.testDir os.environ['TESTDIR'] = self.testDir if os.getenv('WMCORE_KEEP_DIRECTORIES', False): deleteOnDestruction = True logging.info("Generated testDir - %s" % self.testDir) if setTmpDir: os.environ['TMPDIR'] = self.testDir self.deleteTmp = deleteOnDestruction return self.testDir def getBackendFromDbURL(self, dburl): dialectPart = dburl.split(":")[0] if dialectPart == 'mysql': return 'MySQL' elif dialectPart == 'oracle': return 'Oracle' elif dialectPart == 'http': return 'CouchDB' else: raise RuntimeError("Unrecognized dialect %s" % dialectPart) def setDatabaseConnection(self, connectUrl=None, socket=None, destroyAllDatabase = False): """ Set up the database connection by retrieving the environment parameters. The destroyAllDatabase option is for testing ONLY. Never flip that switch on in any other instance where you don't know what you're doing. """ if not self.hasDatabase: return config = self.getConfiguration(connectUrl=connectUrl, socket=socket) self.coreConfig = config self.init.setDatabaseConnection(config.CoreDatabase.connectUrl, config.CoreDatabase.dialect, config.CoreDatabase.socket) if trashDatabases or destroyAllDatabase: self.clearDatabase() # Have to check whether or not database is empty # If the database is not empty when we go to set the schema, abort! try: result = self.init.checkDatabaseContents() except OperationalError: logging.debug("Error checking DB contents, assume DB does not exist") return if len(result) > 0: msg = "Database not empty, cannot set schema !\n" msg += str(result) logging.error(msg) raise TestInitException(msg) return def setSchema(self, customModules = [], useDefault = True, params = None): """ Creates the schema in the database for the default tables/services: trigger, message service, threadpool. Developers can add their own modules to it using the array customModules which should follow the proper naming convention. if useDefault is set to False, it will not instantiate the schemas in the defaultModules array. """ if not self.hasDatabase: return defaultModules = ["WMCore.WMBS"] if not useDefault: defaultModules = [] # filter out unique modules modules = {} for module in (defaultModules + customModules): modules[module] = 'done' try: self.init.setSchema(modules.keys(), params = params) except Exception as ex: print(traceback.format_exc()) raise ex # store the list of modules we've added to the DB modules = {} for module in (defaultModules + customModules + self.currModules): modules[module] = 'done' self.currModules = modules.keys() return def getDBInterface(self): "shouldbe called after connection is made" if not self.hasDatabase: return myThread = threading.currentThread() return myThread.dbi def getConfiguration(self, configurationFile = None, connectUrl = None, socket=None): """ Loads (if available) your configuration file and augments it with the standard settings used in multiple tests. """ if configurationFile != None: config = loadConfigurationFile(configurationFile) else: config = Configuration() # some general settings that would come from the general default # config file config.Agent.contact = "*****@*****.**" config.Agent.teamName = "Lakers" config.Agent.agentName = "Lebron James" config.Agent.hostName = "testhost.laker.world" config.section_("General") # If you need a testDir, call testInit.generateWorkDir # config.General.workDir = os.getenv("TESTDIR") config.section_("CoreDatabase") if connectUrl: config.CoreDatabase.connectUrl = connectUrl config.CoreDatabase.dialect = self.getBackendFromDbURL(connectUrl) config.CoreDatabase.socket = socket or os.getenv("DBSOCK") else: if os.getenv('DATABASE') == None: raise RuntimeError("You must set the DATABASE environment variable to run tests") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.dialect = self.getBackendFromDbURL(os.getenv("DATABASE")) config.CoreDatabase.socket = os.getenv("DBSOCK") if os.getenv("DBHOST"): print("****WARNING: the DBHOST environment variable will be deprecated soon***") print("****WARNING: UPDATE YOUR ENVIRONMENT OR TESTS WILL FAIL****") # after this you can augment it with whatever you need. couchurl = os.getenv("COUCHURL") config.section_("ACDC") config.ACDC.couchurl = couchurl config.ACDC.database = "wmagent_acdc_t" config.component_("JobStateMachine") config.JobStateMachine.couchurl = couchurl config.JobStateMachine.couchDBName = "wmagent_job_test" config.JobStateMachine.jobSummaryDBName = "job_summary" config.JobStateMachine.summaryStatsDBName = "stat_summary_test" config.component_("JobAccountant") config.JobAccountant.pollInterval = 60 config.JobAccountant.componentDir = os.getcwd() config.JobAccountant.logLevel = 'SQLDEBUG' config.component_("TaskArchiver") config.TaskArchiver.localWMStatsURL = "%s/%s" % (config.JobStateMachine.couchurl, config.JobStateMachine.jobSummaryDBName) config.TaskArchiver.ReqMgrSeviceURL = "request manager service url" config.TaskArchiver.ReqMgr2ServiceURL = "https://cmsweb-dev.cern.ch/reqmgr2" return config def clearDatabase(self, modules = []): """ Database deletion. Global, ignore modules. """ if not self.hasDatabase: return self.init.clearDatabase() return def attemptToCloseDBConnections(self): return myThread = threading.currentThread() print("Closing DB") try: if not myThread.transaction \ and not myThread.transaction.conn \ and not myThread.transaction.conn.closed: myThread.transaction.conn.close() myThread.transaction.conn = None print("Connection Closed") except Exception as e: print("tried to close DBI but failed: %s" % e) try: if hasattr(myThread, "dbFactory"): del myThread.dbFactory print("dbFactory removed") except Exception as e: print("tried to delete factory but failed %s" % e)
class TestInit(object): """ A set of initialization steps used in many tests. Test can call the methods from this class to initialize their default environment so to minimize code duplication. """ def __init__(self, testClassName="Unknown Class"): self.testClassName = testClassName self.testDir = None self.currModules = [] global hasDatabase self.hasDatabase = hasDatabase if self.hasDatabase: self.init = WMInit() self.deleteTmp = True def __del__(self): if self.deleteTmp: self.delWorkDir() self.attemptToCloseDBConnections() def delWorkDir(self): if self.testDir != None: try: shutil.rmtree(self.testDir) except: # meh, if it fails, I guess something weird happened pass def setLogging(self, logLevel=logging.INFO): """ Sets logging parameters """ # remove old logging instances. return logger1 = logging.getLogger() logger2 = logging.getLogger(self.testClassName) for logger in [logger1, logger2]: for handler in logger.handlers: handler.close() logger.removeHandler(handler) self.init.setLogging(self.testClassName, self.testClassName, logExists=False, logLevel=logLevel) def generateWorkDir(self, config=None, deleteOnDestruction=True, setTmpDir=False): self.testDir = tempfile.mkdtemp() if config: config.section_("General") config.General.workDir = self.testDir os.environ['TESTDIR'] = self.testDir if os.getenv('WMCORE_KEEP_DIRECTORIES', False): deleteOnDestruction = True logging.info("Generated testDir - %s" % self.testDir) if setTmpDir: os.environ['TMPDIR'] = self.testDir self.deleteTmp = deleteOnDestruction return self.testDir def getBackendFromDbURL(self, dburl): dialectPart = dburl.split(":")[0] if dialectPart == 'mysql': return 'MySQL' elif dialectPart == 'oracle': return 'Oracle' elif dialectPart == 'http': return 'CouchDB' else: raise RuntimeError("Unrecognized dialect %s" % dialectPart) def setDatabaseConnection(self, connectUrl=None, socket=None, destroyAllDatabase=False): """ Set up the database connection by retrieving the environment parameters. The destroyAllDatabase option is for testing ONLY. Never flip that switch on in any other instance where you don't know what you're doing. """ if not self.hasDatabase: return config = self.getConfiguration(connectUrl=connectUrl, socket=socket) self.coreConfig = config self.init.setDatabaseConnection(config.CoreDatabase.connectUrl, config.CoreDatabase.dialect, config.CoreDatabase.socket) if trashDatabases or destroyAllDatabase: self.clearDatabase() # Have to check whether or not database is empty # If the database is not empty when we go to set the schema, abort! try: result = self.init.checkDatabaseContents() except Exception as e: logging.debug( "Error checking DB contents, assume DB does not exist") logging.debug(str(e)) return if len(result) > 0: msg = "Database not empty, cannot set schema !\n" msg += str(result) logging.error(msg) raise TestInitException(msg) return def setSchema(self, customModules=[], useDefault=True, params=None): """ Creates the schema in the database for the default tables/services: trigger, message service, threadpool. Developers can add their own modules to it using the array customModules which should follow the proper naming convention. if useDefault is set to False, it will not instantiate the schemas in the defaultModules array. """ if not self.hasDatabase: return defaultModules = ["WMCore.WMBS"] if not useDefault: defaultModules = [] # filter for unique modules modules = [] for module in (defaultModules + customModules): if module not in modules: modules.append(module) try: self.init.setSchema(modules, params=params) except Exception as ex: print(traceback.format_exc()) raise ex # store the list of modules we've added to the DB # again filter for unique modules modules = [] for module in (defaultModules + customModules + self.currModules): if module not in modules: modules.append(module) self.currModules = modules return def getDBInterface(self): "shouldbe called after connection is made" if not self.hasDatabase: return myThread = threading.currentThread() return myThread.dbi def getConfiguration(self, configurationFile=None, connectUrl=None, socket=None): """ Loads (if available) your configuration file and augments it with the standard settings used in multiple tests. """ if configurationFile != None: config = loadConfigurationFile(configurationFile) else: config = Configuration() # some general settings that would come from the general default # config file config.Agent.contact = "*****@*****.**" config.Agent.teamName = "Lakers" config.Agent.agentName = "Lebron James" config.Agent.hostName = "testhost.laker.world" config.section_("General") # If you need a testDir, call testInit.generateWorkDir # config.General.workDir = os.getenv("TESTDIR") config.General.ReqMgr2ServiceURL = "http://localhost/reqmgr2" config.section_("CoreDatabase") if connectUrl: config.CoreDatabase.connectUrl = connectUrl config.CoreDatabase.dialect = self.getBackendFromDbURL(connectUrl) config.CoreDatabase.socket = socket or os.getenv("DBSOCK") else: if os.getenv('DATABASE') == None: raise RuntimeError( "You must set the DATABASE environment variable to run tests" ) config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.dialect = self.getBackendFromDbURL( os.getenv("DATABASE")) config.CoreDatabase.socket = os.getenv("DBSOCK") if os.getenv("DBHOST"): print( "****WARNING: the DBHOST environment variable will be deprecated soon***" ) print( "****WARNING: UPDATE YOUR ENVIRONMENT OR TESTS WILL FAIL****" ) # after this you can augment it with whatever you need. couchurl = os.getenv("COUCHURL") config.section_("ACDC") config.ACDC.couchurl = couchurl config.ACDC.database = "wmagent_acdc_t" config.component_("JobStateMachine") config.JobStateMachine.couchurl = couchurl config.JobStateMachine.couchDBName = "wmagent_job_test" config.JobStateMachine.jobSummaryDBName = "job_summary" config.JobStateMachine.summaryStatsDBName = "stat_summary_test" config.component_("JobAccountant") config.JobAccountant.pollInterval = 60 config.JobAccountant.componentDir = os.getcwd() config.JobAccountant.logLevel = 'SQLDEBUG' config.component_("TaskArchiver") config.TaskArchiver.localWMStatsURL = "%s/%s" % ( config.JobStateMachine.couchurl, config.JobStateMachine.jobSummaryDBName) return config def clearDatabase(self, modules=[]): """ Database deletion. Global, ignore modules. """ if not self.hasDatabase: return self.init.clearDatabase() return def attemptToCloseDBConnections(self): return myThread = threading.currentThread() print("Closing DB") try: if not myThread.transaction \ and not myThread.transaction.conn \ and not myThread.transaction.conn.closed: myThread.transaction.conn.close() myThread.transaction.conn = None print("Connection Closed") except Exception as e: print("tried to close DBI but failed: %s" % e) try: if hasattr(myThread, "dbFactory"): del myThread.dbFactory print("dbFactory removed") except Exception as e: print("tried to delete factory but failed %s" % e)
class TestInit: """ A set of initialization steps used in many tests. Test can call the methods from this class to initialize their default environment so to minimize code duplication. """ def __init__(self, testClassName = "Unknown Class"): self.testClassName = testClassName self.testDir = None self.currModules = [] global hasDatabase self.hasDatabase = hasDatabase if self.hasDatabase: self.init = WMInit() self.deleteTmp = True def __del__(self): if self.deleteTmp: self.delWorkDir() self.attemptToCloseDBConnections() def delWorkDir(self): if (self.testDir != None): try: shutil.rmtree( self.testDir ) except: # meh, if it fails, I guess something weird happened pass def setLogging(self, logLevel = logging.INFO): """ Sets logging parameters """ # remove old logging instances. return logger1 = logging.getLogger() logger2 = logging.getLogger(self.testClassName) for logger in [logger1, logger2]: for handler in logger.handlers: handler.close() logger.removeHandler(handler) self.init.setLogging(self.testClassName, self.testClassName, logExists = False, logLevel = logLevel) def generateWorkDir(self, config = None, deleteOnDestruction = True, setTmpDir = False): self.testDir = tempfile.mkdtemp() if config: config.section_("General") config.General.workDir = self.testDir os.environ['TESTDIR'] = self.testDir if os.getenv('WMCORE_KEEP_DIRECTORIES', False): deleteOnDestruction = True logging.info("Generated testDir - %s" % self.testDir) if setTmpDir: os.environ['TMPDIR'] = self.testDir self.deleteTmp = deleteOnDestruction return self.testDir def getBackendFromDbURL(self, dburl): dialectPart = dburl.split(":")[0] if dialectPart == 'mysql': return 'MySQL' elif dialectPart == 'sqlite': return 'SQLite' elif dialectPart == 'oracle': return 'Oracle' elif dialectPart == 'http': return 'CouchDB' else: raise RuntimeError, "Unrecognized dialect %s" % dialectPart def setDatabaseConnection(self, connectUrl=None, socket=None, destroyAllDatabase = False): """ Set up the database connection by retrieving the environment parameters. The destroyAllDatabase option is for testing ONLY. Never flip that switch on in any other instance where you don't know what you're doing. """ if not self.hasDatabase: return config = self.getConfiguration(connectUrl=connectUrl, socket=socket) self.coreConfig = config self.init.setDatabaseConnection(config.CoreDatabase.connectUrl, config.CoreDatabase.dialect, config.CoreDatabase.socket) if trashDatabases or destroyAllDatabase: self.clearDatabase() # Have to check whether or not database is empty # If the database is not empty when we go to set the schema, abort! result = self.init.checkDatabaseContents() if len(result) > 0: msg = "Database not empty, cannot set schema !\n" msg += str(result) logging.error(msg) raise TestInitException(msg) return def setSchema(self, customModules = [], useDefault = True, params = None): """ Creates the schema in the database for the default tables/services: trigger, message service, threadpool. Developers can add their own modules to it using the array customModules which should follow the proper naming convention. if useDefault is set to False, it will not instantiate the schemas in the defaultModules array. """ if not self.hasDatabase: return defaultModules = ["WMCore.WMBS"] if not useDefault: defaultModules = [] # filter out unique modules modules = {} for module in (defaultModules + customModules): modules[module] = 'done' try: self.init.setSchema(modules.keys(), params = params) except Exception, ex: print traceback.format_exc() raise ex # store the list of modules we've added to the DB modules = {} for module in (defaultModules + customModules + self.currModules): modules[module] = 'done' self.currModules = modules.keys() return