def execute(self, siteName, conn = None, transaction = False):
"""
_execute_
Determine if the sites already exist in the dbsbuffer_location and
insert any sites that do not already exist in the location table and
return the IDs of all sites that were passed to this function.
The sites will be returned as a dictionary where each key is the site
name and the value is the site ID.
This DAO will create it's own transaction and execute all SQL in that.
This is done so that other transactions can pickup news sites and to
avoid deadlocks.
"""
mySites = copy.deepcopy(siteName)
nameMap = {}
if type(mySites) == str:
mySites = [mySites]
myTransaction = Transaction(self.dbi)
myTransaction.begin()
binds = []
for location in mySites:
binds.append({"location": location})
results = self.dbi.processData(self.existsSQL, binds,
conn = myTransaction.conn,
transaction = True)
results = self.format(results)
for result in results:
nameMap[result[0]] = int(result[1])
mySites.remove(result[0])
binds = []
for location in mySites:
binds.append({"location": location})
if len(binds) > 0:
try:
self.dbi.processData(self.sql, binds, conn = myTransaction.conn,
transaction = True)
except Exception, ex:
if "orig" in dir(ex) and type(ex.orig) != tuple:
if str(ex.orig).find("ORA-00001: unique constraint") != -1 and \
str(ex.orig).find("DBSBUFFER_LOCATION_UNIQUE") != -1:
return
raise ex
results = self.dbi.processData(self.existsSQL, binds,
conn = myTransaction.conn,
transaction = True)
results = self.format(results)
for result in results:
nameMap[result[0]] = int(result[1])
def execute(self, siteName, conn = None, transaction = False):
"""
_execute_
Determine if the sites already exist in the dbsbuffer_location and
attempt to lock the table using a "FOR UPDATE" parameters on the
select statement. Insert any sites that do not already exist in the
location table and return the IDs of all sites that were passed to this
function.
The sites will be returned as a dictionary where each key is the site
name and the value is the site ID.
This DAO will create it's own transaction and execute all SQL in that.
This is done so that other transactions can pickup news sites and to
avoid deadlocks.
"""
mySites = copy.deepcopy(siteName)
nameMap = {}
if type(mySites) == str:
mySites = [mySites]
myTransaction = Transaction(self.dbi)
myTransaction.begin()
binds = []
for location in mySites:
binds.append({"location": location})
results = self.dbi.processData(self.existsSQL, binds,
conn = myTransaction.conn,
transaction = True)
results = self.format(results)
for result in results:
nameMap[result[0]] = int(result[1])
mySites.remove(result[0])
binds = []
for location in mySites:
binds.append({"location": location})
if len(binds) > 0:
self.dbi.processData(self.sql, binds, conn = myTransaction.conn,
transaction = True)
results = self.dbi.processData(self.existsSQL, binds,
conn = myTransaction.conn,
transaction = True)
results = self.format(results)
for result in results:
nameMap[result[0]] = int(result[1])
myTransaction.commit()
return nameMap
def setDatabaseConnection(self, dbConfig, dialect, socketLoc=None):
"""
Sets the default connection parameters, without having to worry
much on what attributes need to be set. This is esepcially
advantagous for developers of third party projects that want
to use only parts of the WMCore lib.
The class differentiates between different formats used by external
projects. External project formats that are supported can activated
it by setting the flavor flag.
"""
myThread = threading.currentThread()
if getattr(myThread, "dialect", None) != None:
# Database is already initialized, we'll create a new
# transaction and move on.
if hasattr(myThread, "transaction"):
if myThread.transaction != None:
myThread.transaction.commit()
myThread.transaction = Transaction(myThread.dbi)
return
options = {}
if dialect.lower() == 'mysql':
dialect = 'MySQL'
if socketLoc != None:
options['unix_socket'] = socketLoc
elif dialect.lower() == 'oracle':
dialect = 'Oracle'
elif dialect.lower() == 'http':
dialect = 'CouchDB'
else:
msg = "Unsupported dialect %s !" % dialect
logging.error(msg)
raise WMInitException(msg)
myThread.dialect = dialect
myThread.logger = logging
myThread.dbFactory = DBFactory(logging, dbConfig, options)
myThread.dbi = myThread.dbFactory.connect()
# The transaction object will begin a transaction as soon as it is
# initialized. I'd rather have the user handle that, so we'll commit
# it here.
myThread.transaction = Transaction(myThread.dbi)
myThread.transaction.commit()
return
def initInThread(self, parameters):
"""
Called when the thread is actually running in its own thread. Performs
internal object setup.
"""
# Get the DB Factory we were passed by parent thread and assign to this
# thread
myThread = threading.currentThread()
myThread.name = self.__class__.__name__
myThread.dbFactory = self.dbFactory
# Now we're in our own thread, set the logger
myThread.logger = self.logger
(connectDialect, _junk) = self.component.config.CoreDatabase.connectUrl.split(":", 1)
if connectDialect.lower() == "mysql":
myThread.dialect = "MySQL"
elif connectDialect.lower() == "oracle":
myThread.dialect = "Oracle"
logging.info("Initialising default database")
myThread.dbi = myThread.dbFactory.connect()
logging.info("Initialising default transaction")
myThread.transaction = Transaction(myThread.dbi)
self.setUpHeartbeat(myThread)
self.setUpLogDB(myThread)
# Call worker setup
self.setup(parameters)
myThread.transaction.commit()
def __init__(self, config = {}):
"""
__DatabasePage__
A page with a database connection (a WMCore.Database.DBFormatter) held
in self.dbi. Look at the DBFormatter class for other handy helper
methods, such as getBinds and formatDict.
The DBFormatter class was originally intended to be extensively
sub-classed, such that it's subclasses followed the DAO pattern. For web
tools we do not generally do this, and you will normally access the
database interface directly:
binds = {'id': 123}
sql = "select * from table where id = :id"
result = self.dbi.processData(sql, binds)
return self.formatDict(result)
Although following the DAO pattern is still possible and encouraged
where appropriate. However, if you want to use the DAO pattern it may be
better to *not* expose the DAO classes and have a normal DatabasePage
exposed that passes the database connection to all the DAO's.
"""
TemplatedPage.__init__(self, config)
dbConfig = ConfigDBMap(config)
conn = DBFactory(self, dbConfig.getDBUrl(), dbConfig.getOption()).connect()
DBFormatter.__init__(self, self, conn)
myThread = threading.currentThread()
myThread.transaction = Transaction(conn)
myThread.transaction.commit()
return
def setUp(self):
"make a logger instance and create tables"
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema()
myThread = threading.currentThread()
if myThread.dialect == 'MySQL':
myThread.create = """
create table test (bind1 varchar(20), bind2 varchar(20)) ENGINE=InnoDB """
if myThread.dialect == 'SQLite':
myThread.create = """
create table test (bind1 varchar(20), bind2 varchar(20))"""
myThread.insert = """
insert into test (bind1, bind2) values (:bind1, :bind2) """
myThread.insert_binds = \
[ {'bind1':'value1a', 'bind2': 'value2a'},\
{'bind1':'value1b', 'bind2': 'value2b'},\
{'bind1':'value1c', 'bind2': 'value2d'} ]
myThread.select = "select * from test"
myThread = threading.currentThread()
myThread.transaction = Transaction(myThread.dbi)
myThread.transaction.processData(myThread.create)
myThread.transaction.processData(myThread.insert,
myThread.insert_binds)
myThread.transaction.commit()
return
def __init__(self, daoPackage, logger=None, dbi=None):
"""
___init___
Initialize all the database connection attributes and the logging
attritbutes. Create a DAO factory for given daoPackage as well. Finally,
check to see if a transaction object has been created. If none exists,
create one but leave the transaction closed.
"""
myThread = threading.currentThread()
if logger:
self.logger = logger
else:
self.logger = myThread.logger
if dbi:
self.dbi = dbi
else:
self.dbi = myThread.dbi
self.daofactory = DAOFactory(package=daoPackage,
logger=self.logger,
dbinterface=self.dbi)
if "transaction" not in dir(myThread):
myThread.transaction = Transaction(self.dbi)
return
def tearDown(self):
"""
Delete the databases
"""
myThread = threading.currentThread()
myThread.transaction = Transaction(myThread.dbi)
myThread.transaction.processData("drop table test")
myThread.transaction.commit()
self.testInit.clearDatabase()
def execute(self, siteName, conn=None, transaction=False):
"""
_execute_
Determine if the sites already exist in the dbsbuffer_location and
attempt to lock the table using a "FOR UPDATE" parameters on the
select statement. Insert any sites that do not already exist in the
location table and return the IDs of all sites that were passed to this
function.
The sites will be returned as a dictionary where each key is the site
name and the value is the site ID.
This DAO will create it's own transaction and execute all SQL in that.
This is done so that other transactions can pickup news sites and to
avoid deadlocks.
"""
mySites = copy.deepcopy(siteName)
nameMap = {}
if type(mySites) == str:
mySites = [mySites]
myTransaction = Transaction(self.dbi)
myTransaction.begin()
binds = []
for location in mySites:
binds.append({"location": location})
results = self.dbi.processData(self.existsSQL,
binds,
conn=myTransaction.conn,
transaction=True)
results = self.format(results)
for result in results:
nameMap[result[0]] = int(result[1])
mySites.remove(result[0])
binds = []
for location in mySites:
binds.append({"location": location})
if len(binds) > 0:
self.dbi.processData(self.sql,
binds,
conn=myTransaction.conn,
transaction=True)
results = self.dbi.processData(self.existsSQL,
binds,
conn=myTransaction.conn,
transaction=True)
results = self.format(results)
for result in results:
nameMap[result[0]] = int(result[1])
myTransaction.commit()
return nameMap
def testListSitesTransaction(self):
"""
_testListSitesTransaction_
Verify that select behave appropriately when dealing with transactions.
"""
myThread = threading.currentThread()
daoFactory = DAOFactory(package="WMCore.WMBS", logger = myThread.logger,
dbinterface = myThread.dbi)
myThread.transaction.begin()
localTransaction = Transaction(myThread.dbi)
localTransaction.begin()
locationNew = daoFactory(classname = "Locations.New")
locationNew.execute("Satsuma", conn = myThread.transaction.conn, transaction = True)
locationNew.execute("Choshu", conn = myThread.transaction.conn, transaction = True)
listSites = daoFactory(classname = "Locations.ListSites")
nonTransSites = listSites.execute(conn = localTransaction.conn, transaction = True)
transSites = listSites.execute(conn = myThread.transaction.conn, transaction = True)
assert len(nonTransSites) == 0, \
"Error: Wrong number of sites in non transaction list."
assert len(transSites) == 2, \
"Error: Wrong number of sites in transaction list."
assert "Satsuma" in transSites, \
"Error: Site missing in transaction list."
assert "Choshu" in transSites, \
"Error: Site missing in transaction list."
localTransaction.commit()
myThread.transaction.commit()
return
def execute(self, siteName, conn = None, transaction = False):
if type(siteName) == str:
binds = {"location": siteName}
else:
binds = []
for aLocation in siteName:
binds.append({"location": aLocation})
myTransaction = Transaction(self.dbi)
myTransaction.begin()
nameMap = {}
self.dbi.processData(self.sql, binds, conn = conn,
transaction = transaction)
results = self.dbi.processData(self.existsSQL, binds,
conn = myTransaction.conn,
transaction = True)
results = self.format(results)
for result in results:
nameMap[result[0]] = int(result[1])
myTransaction.commit()
return nameMap
def beginTransaction(self):
"""
_beginTransaction_
Begin a database transaction if one does not already exist.
"""
myThread = threading.currentThread()
if "transaction" not in dir(myThread):
myThread.transaction = Transaction(self.dbi)
return False
if myThread.transaction.transaction == None:
myThread.transaction.begin()
return False
return True
def testD(self):
raise nose.SkipTest
config = self.testInit.getConfiguration()
config.component_("TestComponent")
config.TestComponent.logLevel = 'INFO'
config.section_("General")
self.tempDir = self.testInit.generateWorkDir(config)
# try starting a component as a daemon:
config.TestComponent.componentDir = os.path.join( \
self.tempDir, "Components/TestComponent2")
os.makedirs(config.TestComponent.componentDir)
testComponent = TestComponent(config)
# we set the parent to true as we are testing
testComponent.startDaemon(keepParent=True)
time.sleep(2)
daemonFile = os.path.join(config.TestComponent.componentDir,
"Daemon.xml")
details = Details(daemonFile)
print('Is component alive: ' + str(details.isAlive()))
# create msgService to send stop message.
myThread = threading.currentThread()
factory = WMFactory("msgService", "WMCore.MsgService." + \
myThread.dialect)
myThread.transaction = Transaction(myThread.dbi)
msgService = factory.loadObject("MsgService")
msgService.registerAs("HarnessTest")
myThread.transaction.commit()
print(
'Publish a stop message to test if the component shutsdown gracefully'
)
myThread.transaction.begin()
msg = {'name': 'Stop', 'payload': ''}
msgService.publish(msg)
myThread.transaction.commit()
msgService.finish()
while details.isAlive():
print('Component has not received stop message')
time.sleep(2)
print('Daemon shutdown gracefully')
def initInThread(self):
"""
ThreadSlave (subclasses) are returned to a queue when no work
is available and can be activated by a new thread when work
is entered in the queue. This method is called when the ThreadSlave
(subclass) is associated to a new thread.
"""
# we need to call this method only when it is called within a thread
# otherwise these parameters are not accissible in the thread used
# to call this threadslave.
myThread = threading.currentThread()
# we are now in our own thread, so we can pass the dbFactory reference
# to here:
myThread.dbFactory = self.dbFactory
if self.component.config.CoreDatabase.dialect:
myThread.dialect = self.component.config.CoreDatabase.dialect
else:
# FIXME we aren't using the DIALECT environment variable anymore
myThread.dialect = os.getenv("DIALECT")
#TODO: remove as much as possible logging statements or make them debug
myThread.logger = logging.getLogger()
logging.info("THREAD: Initializing default database")
logging.info("THREAD: Check if connection is through socket")
logging.info("THREAD: Building database connection string")
# we ensured that we use the dbFactory object from our parent
# thread so we have only one engine in the application.
myThread.dbi = myThread.dbFactory.connect()
logging.info("THREAD: Initializing default transaction")
myThread.transaction = Transaction(myThread.dbi)
logging.info("THREAD: Loading backend")
logging.info("THREAD constructor finished")
def testListSitesTransaction(self):
"""
_testListSitesTransaction_
Verify that select behave appropriately when dealing with transactions.
"""
myThread = threading.currentThread()
daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
myThread.transaction.begin()
localTransaction = Transaction(myThread.dbi)
localTransaction.begin()
locationNew = daoFactory(classname="Locations.New")
locationNew.execute("Satsuma",
conn=myThread.transaction.conn,
transaction=True)
locationNew.execute("Choshu",
conn=myThread.transaction.conn,
transaction=True)
listSites = daoFactory(classname="Locations.ListSites")
nonTransSites = listSites.execute(conn=localTransaction.conn,
transaction=True)
transSites = listSites.execute(conn=myThread.transaction.conn,
transaction=True)
assert len(nonTransSites) == 0, \
"Error: Wrong number of sites in non transaction list."
assert len(transSites) == 2, \
"Error: Wrong number of sites in transaction list."
assert "Satsuma" in transSites, \
"Error: Site missing in transaction list."
assert "Choshu" in transSites, \
"Error: Site missing in transaction list."
localTransaction.commit()
myThread.transaction.commit()
return
def initInThread(self):
"""
Default intialization of the harness including setting some diagnostic
messages. This method is called when we call 'prepareToStart'
"""
try:
self.messages = {}
compName = self.config.Agent.componentName
compSect = getattr(self.config, compName, None)
if not hasattr(compSect, "logFile"):
if not getattr(compSect, 'componentDir', None):
errorMessage = "No componentDir for log entries found!\n"
errorMessage += "Harness cannot run without componentDir.\n"
logging.error(errorMessage)
raise HarnessException(errorMessage)
compSect.logFile = os.path.join(compSect.componentDir,
"ComponentLog")
print('Log file is: ' + compSect.logFile)
logHandler = RotatingFileHandler(compSect.logFile, "a", 1000000000,
3)
logMsgFormat = getattr(
compSect, "logMsgFormat",
"%(asctime)s:%(thread)d:%(levelname)s:%(module)s:%(message)s")
logFormatter = \
logging.Formatter(logMsgFormat)
logHandler.setFormatter(logFormatter)
logLevelName = getattr(compSect, 'logLevel', 'INFO')
logLevel = getattr(logging, logLevelName)
logging.getLogger().addHandler(logHandler)
logging.getLogger().setLevel(logLevel)
self.logMsg = {
'DEBUG': logging.DEBUG,
'ERROR': logging.ERROR,
'NOTSET': logging.NOTSET,
'CRITICAL': logging.CRITICAL,
'WARNING': logging.WARNING,
'INFO': logging.INFO,
'SQLDEBUG': logging.SQLDEBUG
}
if hasattr(compSect,
"logLevel") and compSect.logLevel in self.logMsg:
logging.getLogger().setLevel(self.logMsg[compSect.logLevel])
WMLogging.sqldebug("wmcore level debug:")
# If not previously set, force wmcore cache to current path
if not os.environ.get('WMCORE_CACHE_DIR'):
os.environ['WMCORE_CACHE_DIR'] = os.path.join(
compSect.componentDir, '.wmcore_cache')
logging.info(">>>Starting: " + compName + '<<<')
# check which backend to use: MySQL, Oracle, etc... for core
# services.
# we recognize there can be more than one database.
# be we offer a default database that is used for core services.
logging.info(">>>Initializing default database")
logging.info(">>>Check if connection is through socket")
myThread = threading.currentThread()
myThread.logger = logging.getLogger()
logging.info(">>>Setting config for thread: ")
myThread.config = self.config
logging.info(">>>Building database connection string")
# check if there is a premade string if not build it yourself.
dbConfig = ConfigDBMap(self.config)
dbStr = dbConfig.getDBUrl()
options = dbConfig.getOption()
# we only want one DBFactory per database so we will need to
# to pass this on in case we are using threads.
myThread.dbFactory = DBFactory(myThread.logger, dbStr, options)
myThread.sql_transaction = True
if myThread.dbFactory.engine:
myThread.dbi = myThread.dbFactory.connect()
myThread.transaction = Transaction(myThread.dbi)
else:
myThread.dbi = myThread.config.CoreDatabase.connectUrl
myThread.sql_transaction = False
# Attach a worker manager object to the main thread
if not hasattr(myThread, 'workerThreadManager'):
myThread.workerThreadManager = WorkerThreadManager(self)
else:
myThread.workerThreadManager.terminateSlaves.clear()
myThread.workerThreadManager.pauseWorkers()
logging.info(">>>Initialize transaction dictionary")
(connectDialect, dummy) = dbStr.split(":", 1)
if connectDialect.lower() == 'mysql':
myThread.dialect = 'MySQL'
elif connectDialect.lower() == 'oracle':
myThread.dialect = 'Oracle'
logging.info("Harness part constructor finished")
except Exception as ex:
logging.critical("Problem instantiating " + str(ex))
logging.error("Traceback: %s", str(traceback.format_exc()))
raise