def testOracleDatabase(self):
"""
Testing Oracle basic operations
"""
dialect = os.environ.get("DIALECT", "MySQL")
if dialect.lower() == 'mysql':
# this test can only run for Oracle
return
init = WMInit()
url = os.environ.get("DATABASE")
init.setDatabaseConnection(url, dialect)
selectDbName = "SELECT ora_database_name FROM DUAL"
destroyDb = """DECLARE
BEGIN
execute immediate 'purge recyclebin';
-- Tables
FOR o IN (SELECT table_name name FROM user_tables) LOOP
execute immediate 'drop table ' || o.name || ' cascade constraints';
END LOOP;
-- Sequences
FOR o IN (SELECT sequence_name name FROM user_sequences) LOOP
execute immediate 'drop sequence ' || o.name;
END LOOP;
-- Triggers
FOR o IN (SELECT trigger_name name FROM user_triggers) LOOP
execute immediate 'drop trigger ' || o.name;
END LOOP;
-- Synonyms
FOR o IN (SELECT synonym_name name FROM user_synonyms) LOOP
execute immediate 'drop synonym ' || o.name;
END LOOP;
-- Functions
FOR o IN (SELECT object_name name FROM user_objects WHERE object_type = 'FUNCTION') LOOP
execute immediate 'drop function ' || o.name;
END LOOP;
-- Procedures
FOR o IN (SELECT object_name name FROM user_objects WHERE object_type = 'PROCEDURE') LOOP
execute immediate 'drop procedure ' || o.name;
END LOOP;
execute immediate 'purge recyclebin';
END;"""
try:
# Initial clear should work
myThread = threading.currentThread()
init.clearDatabase()
# Clear non-existant DB should work
init.clearDatabase()
init.setSchema(modules=['WMCore.WMBS'])
# Drop the database, and then make sure the database gets recreated
a = myThread.dbi.engine.url.database
self.assertEqual(myThread.dbi.engine.name, "oracle")
self.assertIsNone(myThread.dbi.engine.url.database)
self.assertEqual(myThread.dbi.engine.url.get_backend_name(),
"oracle")
self.assertEqual(myThread.dbi.engine.url.get_driver_name(),
"cx_oracle")
self.assertEqual(myThread.dbi.engine.url.host, "INT2R_NOLB")
dbName = myThread.dbi.processData(selectDbName)[0].fetchall()[0][0]
self.assertTrue(dbName)
myThread.transaction.processData("SELECT * FROM wmbs_job")
init.clearDatabase()
dbName = myThread.dbi.processData(selectDbName)[0].fetchall()[0][0]
self.assertTrue(dbName)
myThread.dbi.processData(destroyDb)
init.setSchema(modules=['WMCore.WMBS'])
myThread.transaction.begin()
myThread.transaction.processData("SELECT * FROM wmbs_job")
init.clearDatabase()
dbName = myThread.dbi.processData(selectDbName)[0].fetchall()[0][0]
self.assertTrue(dbName)
myThread.transaction.begin()
init.setSchema(modules=['WMCore.WMBS'])
myThread.transaction.commit()
except:
init.clearDatabase()
raise
else:
init.clearDatabase()
def testMySQLDatabase(self):
"""
Testing MySQL basic operations
"""
dialect = os.environ.get("DIALECT", "MySQL")
if dialect.lower() == 'oracle':
# this test can only run for MySQL
return
init = WMInit()
url = os.environ.get("DATABASE")
sock = os.environ.get("DBSOCK", None)
init.setDatabaseConnection(url, dialect, sock)
selectDbName = "SELECT DATABASE() AS dbname"
destroyDbName = "DROP DATABASE %s"
try:
# Initial clear should work
myThread = threading.currentThread()
init.clearDatabase()
# Clear non-existant DB should work
init.clearDatabase()
init.setSchema(modules=['WMCore.WMBS'])
# Drop the database, and then make sure the database gets recreated
a = myThread.dbi.engine.url.database
self.assertEqual(myThread.dbi.engine.name, "mysql")
self.assertTrue(
myThread.dbi.engine.url.database in ("wmcore_unittest",
"WMCore_unit_test"))
self.assertEqual(myThread.dbi.engine.url.get_backend_name(),
"mysql")
self.assertEqual(myThread.dbi.engine.url.get_driver_name(),
"mysqldb")
self.assertEqual(myThread.dbi.engine.url.host, "localhost")
dbName = myThread.dbi.processData(selectDbName)[0].fetchall()[0][0]
self.assertEqual(dbName, a)
myThread.dbi.processData(destroyDbName % dbName)
dbName = myThread.dbi.processData(selectDbName)[0].fetchall()[0][0]
self.assertEqual(dbName, None)
init.clearDatabase()
dbName = myThread.dbi.processData(selectDbName)[0].fetchall()[0][0]
self.assertEqual(dbName, a)
init.setSchema(modules=['WMCore.WMBS'])
myThread.transaction.begin()
myThread.transaction.processData("SELECT * FROM wmbs_job")
init.clearDatabase()
dbName = myThread.dbi.processData(selectDbName)[0].fetchall()[0][0]
self.assertEqual(dbName, a)
myThread.transaction.begin()
init.setSchema(modules=['WMCore.WMBS'])
myThread.transaction.commit()
except:
init.clearDatabase()
raise
else:
init.clearDatabase()
return
class TestInit(object):
"""
A set of initialization steps used in many tests.
Test can call the methods from this class to
initialize their default environment so to
minimize code duplication.
"""
def __init__(self, testClassName="Unknown Class"):
self.testClassName = testClassName
self.testDir = None
self.currModules = []
global hasDatabase
self.hasDatabase = hasDatabase
if self.hasDatabase:
self.init = WMInit()
self.deleteTmp = True
def __del__(self):
if self.deleteTmp:
self.delWorkDir()
self.attemptToCloseDBConnections()
def delWorkDir(self):
if self.testDir != None:
try:
shutil.rmtree(self.testDir)
except:
# meh, if it fails, I guess something weird happened
pass
def setLogging(self, logLevel=logging.INFO):
"""
Sets logging parameters
"""
# remove old logging instances.
return
logger1 = logging.getLogger()
logger2 = logging.getLogger(self.testClassName)
for logger in [logger1, logger2]:
for handler in logger.handlers:
handler.close()
logger.removeHandler(handler)
self.init.setLogging(self.testClassName,
self.testClassName,
logExists=False,
logLevel=logLevel)
def generateWorkDir(self,
config=None,
deleteOnDestruction=True,
setTmpDir=False):
self.testDir = tempfile.mkdtemp()
if config:
config.section_("General")
config.General.workDir = self.testDir
os.environ['TESTDIR'] = self.testDir
if os.getenv('WMCORE_KEEP_DIRECTORIES', False):
deleteOnDestruction = True
logging.info("Generated testDir - %s" % self.testDir)
if setTmpDir:
os.environ['TMPDIR'] = self.testDir
self.deleteTmp = deleteOnDestruction
return self.testDir
def getBackendFromDbURL(self, dburl):
dialectPart = dburl.split(":")[0]
if dialectPart == 'mysql':
return 'MySQL'
elif dialectPart == 'oracle':
return 'Oracle'
elif dialectPart == 'http':
return 'CouchDB'
else:
raise RuntimeError("Unrecognized dialect %s" % dialectPart)
def setDatabaseConnection(self,
connectUrl=None,
socket=None,
destroyAllDatabase=False):
"""
Set up the database connection by retrieving the environment
parameters.
The destroyAllDatabase option is for testing ONLY. Never flip that switch
on in any other instance where you don't know what you're doing.
"""
if not self.hasDatabase:
return
config = self.getConfiguration(connectUrl=connectUrl, socket=socket)
self.coreConfig = config
self.init.setDatabaseConnection(config.CoreDatabase.connectUrl,
config.CoreDatabase.dialect,
config.CoreDatabase.socket)
if trashDatabases or destroyAllDatabase:
self.clearDatabase()
# Have to check whether or not database is empty
# If the database is not empty when we go to set the schema, abort!
try:
result = self.init.checkDatabaseContents()
except Exception as e:
logging.debug(
"Error checking DB contents, assume DB does not exist")
logging.debug(str(e))
return
if len(result) > 0:
msg = "Database not empty, cannot set schema !\n"
msg += str(result)
logging.error(msg)
raise TestInitException(msg)
return
def setSchema(self, customModules=[], useDefault=True, params=None):
"""
Creates the schema in the database for the default
tables/services: trigger, message service, threadpool.
Developers can add their own modules to it using the array
customModules which should follow the proper naming convention.
if useDefault is set to False, it will not instantiate the
schemas in the defaultModules array.
"""
if not self.hasDatabase:
return
defaultModules = ["WMCore.WMBS"]
if not useDefault:
defaultModules = []
# filter for unique modules
modules = []
for module in (defaultModules + customModules):
if module not in modules:
modules.append(module)
try:
self.init.setSchema(modules, params=params)
except Exception as ex:
print(traceback.format_exc())
raise ex
# store the list of modules we've added to the DB
# again filter for unique modules
modules = []
for module in (defaultModules + customModules + self.currModules):
if module not in modules:
modules.append(module)
self.currModules = modules
return
def getDBInterface(self):
"shouldbe called after connection is made"
if not self.hasDatabase:
return
myThread = threading.currentThread()
return myThread.dbi
def getConfiguration(self,
configurationFile=None,
connectUrl=None,
socket=None):
"""
Loads (if available) your configuration file and augments
it with the standard settings used in multiple tests.
"""
if configurationFile != None:
config = loadConfigurationFile(configurationFile)
else:
config = Configuration()
# some general settings that would come from the general default
# config file
config.Agent.contact = "*****@*****.**"
config.Agent.teamName = "Lakers"
config.Agent.agentName = "Lebron James"
config.Agent.hostName = "testhost.laker.world"
config.section_("General")
# If you need a testDir, call testInit.generateWorkDir
# config.General.workDir = os.getenv("TESTDIR")
config.General.ReqMgr2ServiceURL = "http://localhost/reqmgr2"
config.section_("CoreDatabase")
if connectUrl:
config.CoreDatabase.connectUrl = connectUrl
config.CoreDatabase.dialect = self.getBackendFromDbURL(connectUrl)
config.CoreDatabase.socket = socket or os.getenv("DBSOCK")
else:
if os.getenv('DATABASE') == None:
raise RuntimeError(
"You must set the DATABASE environment variable to run tests"
)
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.dialect = self.getBackendFromDbURL(
os.getenv("DATABASE"))
config.CoreDatabase.socket = os.getenv("DBSOCK")
if os.getenv("DBHOST"):
print(
"****WARNING: the DBHOST environment variable will be deprecated soon***"
)
print(
"****WARNING: UPDATE YOUR ENVIRONMENT OR TESTS WILL FAIL****"
)
# after this you can augment it with whatever you need.
couchurl = os.getenv("COUCHURL")
config.section_("ACDC")
config.ACDC.couchurl = couchurl
config.ACDC.database = "wmagent_acdc_t"
config.component_("JobStateMachine")
config.JobStateMachine.couchurl = couchurl
config.JobStateMachine.couchDBName = "wmagent_job_test"
config.JobStateMachine.jobSummaryDBName = "job_summary"
config.JobStateMachine.summaryStatsDBName = "stat_summary_test"
config.component_("JobAccountant")
config.JobAccountant.pollInterval = 60
config.JobAccountant.componentDir = os.getcwd()
config.JobAccountant.logLevel = 'SQLDEBUG'
config.component_("TaskArchiver")
config.TaskArchiver.localWMStatsURL = "%s/%s" % (
config.JobStateMachine.couchurl,
config.JobStateMachine.jobSummaryDBName)
return config
def clearDatabase(self, modules=[]):
"""
Database deletion. Global, ignore modules.
"""
if not self.hasDatabase:
return
self.init.clearDatabase()
return
def attemptToCloseDBConnections(self):
return
myThread = threading.currentThread()
print("Closing DB")
try:
if not myThread.transaction \
and not myThread.transaction.conn \
and not myThread.transaction.conn.closed:
myThread.transaction.conn.close()
myThread.transaction.conn = None
print("Connection Closed")
except Exception as e:
print("tried to close DBI but failed: %s" % e)
try:
if hasattr(myThread, "dbFactory"):
del myThread.dbFactory
print("dbFactory removed")
except Exception as e:
print("tried to delete factory but failed %s" % e)
def testB_Database(self):
"""
_Database_
Testing the database stuff. Only works for MySQL backend
"""
init = WMInit()
url = os.environ.get("DATABASE")
dialect = os.environ.get("DIALECT", "MySQL")
sock = os.environ.get("DBSOCK", None)
init.setDatabaseConnection(url, dialect, sock)
try:
# Initial clear should work
myThread = threading.currentThread()
init.clearDatabase()
# Clear one after another should work
init.setSchema(modules=['WMCore.WMBS'])
init.clearDatabase()
init.setSchema(modules=['WMCore.WMBS'])
init.clearDatabase()
# Clear non-existant DB should work
# Drop the database, and then make sure the database gets recreated
a = myThread.dbi.engine.url.database
dbName = myThread.dbi.processData(
"SELECT DATABASE() AS dbname")[0].fetchall()[0][0]
myThread.dbi.processData("DROP DATABASE %s" % dbName)
dbName = myThread.dbi.processData(
"SELECT DATABASE() AS dbname")[0].fetchall()[0][0]
self.assertEqual(dbName, None)
init.clearDatabase()
dbName = myThread.dbi.processData(
"SELECT DATABASE() AS dbname")[0].fetchall()[0][0]
self.assertEqual(dbName, a)
init.setSchema(modules=['WMCore.WMBS'])
myThread.transaction.begin()
myThread.transaction.processData("SELECT * FROM wmbs_job")
init.clearDatabase()
dbName = myThread.dbi.processData(
"SELECT DATABASE() AS dbname")[0].fetchall()[0][0]
self.assertEqual(dbName, a)
myThread.transaction.begin()
init.setSchema(modules=['WMCore.WMBS'])
myThread.transaction.commit()
except:
init.clearDatabase()
raise
init.clearDatabase()
return
def testB_Database(self):
"""
_Database_
Testing the database stuff.
"""
init = WMInit()
url = os.environ.get("DATABASE")
dialect = os.environ.get("DIALECT")
sock = os.environ.get("DBSOCK", None)
init.setDatabaseConnection(url, dialect, sock)
try:
# Initial clear should work
myThread = threading.currentThread()
init.clearDatabase()
# Clear one after another should work
init.setSchema(modules = ['WMCore.WMBS'])
init.clearDatabase()
init.setSchema(modules = ['WMCore.WMBS'])
init.clearDatabase()
# Clear non-existant DB should work
# Drop the database, and then make sure the database gets recreated
a = myThread.dbi.engine.url.database
dbName = myThread.dbi.processData("SELECT DATABASE() AS dbname")[0].fetchall()[0][0]
myThread.dbi.processData("DROP DATABASE %s" % dbName)
dbName = myThread.dbi.processData("SELECT DATABASE() AS dbname")[0].fetchall()[0][0]
self.assertEqual(dbName, None)
init.clearDatabase()
dbName = myThread.dbi.processData("SELECT DATABASE() AS dbname")[0].fetchall()[0][0]
self.assertEqual(dbName, a)
init.setSchema(modules = ['WMCore.WMBS'])
myThread.transaction.begin()
myThread.transaction.processData("SELECT * FROM wmbs_job")
init.clearDatabase()
dbName = myThread.dbi.processData("SELECT DATABASE() AS dbname")[0].fetchall()[0][0]
self.assertEqual(dbName, a)
myThread.transaction.begin()
init.setSchema(modules = ['WMCore.WMBS'])
myThread.transaction.commit()
except:
init.clearDatabase()
raise
init.clearDatabase()
return
class TestInit(object):
"""
A set of initialization steps used in many tests.
Test can call the methods from this class to
initialize their default environment so to
minimize code duplication.
"""
def __init__(self, testClassName = "Unknown Class"):
self.testClassName = testClassName
self.testDir = None
self.currModules = []
global hasDatabase
self.hasDatabase = hasDatabase
if self.hasDatabase:
self.init = WMInit()
self.deleteTmp = True
def __del__(self):
if self.deleteTmp:
self.delWorkDir()
self.attemptToCloseDBConnections()
def delWorkDir(self):
if self.testDir != None:
try:
shutil.rmtree( self.testDir )
except:
# meh, if it fails, I guess something weird happened
pass
def setLogging(self, logLevel = logging.INFO):
"""
Sets logging parameters
"""
# remove old logging instances.
return
logger1 = logging.getLogger()
logger2 = logging.getLogger(self.testClassName)
for logger in [logger1, logger2]:
for handler in logger.handlers:
handler.close()
logger.removeHandler(handler)
self.init.setLogging(self.testClassName, self.testClassName,
logExists = False, logLevel = logLevel)
def generateWorkDir(self, config = None, deleteOnDestruction = True, setTmpDir = False):
self.testDir = tempfile.mkdtemp()
if config:
config.section_("General")
config.General.workDir = self.testDir
os.environ['TESTDIR'] = self.testDir
if os.getenv('WMCORE_KEEP_DIRECTORIES', False):
deleteOnDestruction = True
logging.info("Generated testDir - %s" % self.testDir)
if setTmpDir:
os.environ['TMPDIR'] = self.testDir
self.deleteTmp = deleteOnDestruction
return self.testDir
def getBackendFromDbURL(self, dburl):
dialectPart = dburl.split(":")[0]
if dialectPart == 'mysql':
return 'MySQL'
elif dialectPart == 'oracle':
return 'Oracle'
elif dialectPart == 'http':
return 'CouchDB'
else:
raise RuntimeError("Unrecognized dialect %s" % dialectPart)
def setDatabaseConnection(self, connectUrl=None, socket=None, destroyAllDatabase = False):
"""
Set up the database connection by retrieving the environment
parameters.
The destroyAllDatabase option is for testing ONLY. Never flip that switch
on in any other instance where you don't know what you're doing.
"""
if not self.hasDatabase:
return
config = self.getConfiguration(connectUrl=connectUrl, socket=socket)
self.coreConfig = config
self.init.setDatabaseConnection(config.CoreDatabase.connectUrl,
config.CoreDatabase.dialect,
config.CoreDatabase.socket)
if trashDatabases or destroyAllDatabase:
self.clearDatabase()
# Have to check whether or not database is empty
# If the database is not empty when we go to set the schema, abort!
try:
result = self.init.checkDatabaseContents()
except OperationalError:
logging.debug("Error checking DB contents, assume DB does not exist")
return
if len(result) > 0:
msg = "Database not empty, cannot set schema !\n"
msg += str(result)
logging.error(msg)
raise TestInitException(msg)
return
def setSchema(self, customModules = [], useDefault = True, params = None):
"""
Creates the schema in the database for the default
tables/services: trigger, message service, threadpool.
Developers can add their own modules to it using the array
customModules which should follow the proper naming convention.
if useDefault is set to False, it will not instantiate the
schemas in the defaultModules array.
"""
if not self.hasDatabase:
return
defaultModules = ["WMCore.WMBS"]
if not useDefault:
defaultModules = []
# filter out unique modules
modules = {}
for module in (defaultModules + customModules):
modules[module] = 'done'
try:
self.init.setSchema(modules.keys(), params = params)
except Exception as ex:
print(traceback.format_exc())
raise ex
# store the list of modules we've added to the DB
modules = {}
for module in (defaultModules + customModules + self.currModules):
modules[module] = 'done'
self.currModules = modules.keys()
return
def getDBInterface(self):
"shouldbe called after connection is made"
if not self.hasDatabase:
return
myThread = threading.currentThread()
return myThread.dbi
def getConfiguration(self, configurationFile = None, connectUrl = None, socket=None):
"""
Loads (if available) your configuration file and augments
it with the standard settings used in multiple tests.
"""
if configurationFile != None:
config = loadConfigurationFile(configurationFile)
else:
config = Configuration()
# some general settings that would come from the general default
# config file
config.Agent.contact = "*****@*****.**"
config.Agent.teamName = "Lakers"
config.Agent.agentName = "Lebron James"
config.Agent.hostName = "testhost.laker.world"
config.section_("General")
# If you need a testDir, call testInit.generateWorkDir
# config.General.workDir = os.getenv("TESTDIR")
config.section_("CoreDatabase")
if connectUrl:
config.CoreDatabase.connectUrl = connectUrl
config.CoreDatabase.dialect = self.getBackendFromDbURL(connectUrl)
config.CoreDatabase.socket = socket or os.getenv("DBSOCK")
else:
if os.getenv('DATABASE') == None:
raise RuntimeError("You must set the DATABASE environment variable to run tests")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.dialect = self.getBackendFromDbURL(os.getenv("DATABASE"))
config.CoreDatabase.socket = os.getenv("DBSOCK")
if os.getenv("DBHOST"):
print("****WARNING: the DBHOST environment variable will be deprecated soon***")
print("****WARNING: UPDATE YOUR ENVIRONMENT OR TESTS WILL FAIL****")
# after this you can augment it with whatever you need.
couchurl = os.getenv("COUCHURL")
config.section_("ACDC")
config.ACDC.couchurl = couchurl
config.ACDC.database = "wmagent_acdc_t"
config.component_("JobStateMachine")
config.JobStateMachine.couchurl = couchurl
config.JobStateMachine.couchDBName = "wmagent_job_test"
config.JobStateMachine.jobSummaryDBName = "job_summary"
config.JobStateMachine.summaryStatsDBName = "stat_summary_test"
config.component_("JobAccountant")
config.JobAccountant.pollInterval = 60
config.JobAccountant.componentDir = os.getcwd()
config.JobAccountant.logLevel = 'SQLDEBUG'
config.component_("TaskArchiver")
config.TaskArchiver.localWMStatsURL = "%s/%s" % (config.JobStateMachine.couchurl, config.JobStateMachine.jobSummaryDBName)
config.TaskArchiver.ReqMgrSeviceURL = "request manager service url"
config.TaskArchiver.ReqMgr2ServiceURL = "https://cmsweb-dev.cern.ch/reqmgr2"
return config
def clearDatabase(self, modules = []):
"""
Database deletion. Global, ignore modules.
"""
if not self.hasDatabase:
return
self.init.clearDatabase()
return
def attemptToCloseDBConnections(self):
return
myThread = threading.currentThread()
print("Closing DB")
try:
if not myThread.transaction \
and not myThread.transaction.conn \
and not myThread.transaction.conn.closed:
myThread.transaction.conn.close()
myThread.transaction.conn = None
print("Connection Closed")
except Exception as e:
print("tried to close DBI but failed: %s" % e)
try:
if hasattr(myThread, "dbFactory"):
del myThread.dbFactory
print("dbFactory removed")
except Exception as e:
print("tried to delete factory but failed %s" % e)
