def execute(self, siteName, conn = None, transaction = False): """ _execute_ Determine if the sites already exist in the dbsbuffer_location and insert any sites that do not already exist in the location table and return the IDs of all sites that were passed to this function. The sites will be returned as a dictionary where each key is the site name and the value is the site ID. This DAO will create it's own transaction and execute all SQL in that. This is done so that other transactions can pickup news sites and to avoid deadlocks. """ mySites = copy.deepcopy(siteName) nameMap = {} if type(mySites) == str: mySites = [mySites] myTransaction = Transaction(self.dbi) myTransaction.begin() binds = [] for location in mySites: binds.append({"location": location}) results = self.dbi.processData(self.existsSQL, binds, conn = myTransaction.conn, transaction = True) results = self.format(results) for result in results: nameMap[result[0]] = int(result[1]) mySites.remove(result[0]) binds = [] for location in mySites: binds.append({"location": location}) if len(binds) > 0: try: self.dbi.processData(self.sql, binds, conn = myTransaction.conn, transaction = True) except Exception, ex: if "orig" in dir(ex) and type(ex.orig) != tuple: if str(ex.orig).find("ORA-00001: unique constraint") != -1 and \ str(ex.orig).find("DBSBUFFER_LOCATION_UNIQUE") != -1: return raise ex results = self.dbi.processData(self.existsSQL, binds, conn = myTransaction.conn, transaction = True) results = self.format(results) for result in results: nameMap[result[0]] = int(result[1])
def execute(self, siteName, conn = None, transaction = False): """ _execute_ Determine if the sites already exist in the dbsbuffer_location and attempt to lock the table using a "FOR UPDATE" parameters on the select statement. Insert any sites that do not already exist in the location table and return the IDs of all sites that were passed to this function. The sites will be returned as a dictionary where each key is the site name and the value is the site ID. This DAO will create it's own transaction and execute all SQL in that. This is done so that other transactions can pickup news sites and to avoid deadlocks. """ mySites = copy.deepcopy(siteName) nameMap = {} if type(mySites) == str: mySites = [mySites] myTransaction = Transaction(self.dbi) myTransaction.begin() binds = [] for location in mySites: binds.append({"location": location}) results = self.dbi.processData(self.existsSQL, binds, conn = myTransaction.conn, transaction = True) results = self.format(results) for result in results: nameMap[result[0]] = int(result[1]) mySites.remove(result[0]) binds = [] for location in mySites: binds.append({"location": location}) if len(binds) > 0: self.dbi.processData(self.sql, binds, conn = myTransaction.conn, transaction = True) results = self.dbi.processData(self.existsSQL, binds, conn = myTransaction.conn, transaction = True) results = self.format(results) for result in results: nameMap[result[0]] = int(result[1]) myTransaction.commit() return nameMap
def setDatabaseConnection(self, dbConfig, dialect, socketLoc=None): """ Sets the default connection parameters, without having to worry much on what attributes need to be set. This is esepcially advantagous for developers of third party projects that want to use only parts of the WMCore lib. The class differentiates between different formats used by external projects. External project formats that are supported can activated it by setting the flavor flag. """ myThread = threading.currentThread() if getattr(myThread, "dialect", None) != None: # Database is already initialized, we'll create a new # transaction and move on. if hasattr(myThread, "transaction"): if myThread.transaction != None: myThread.transaction.commit() myThread.transaction = Transaction(myThread.dbi) return options = {} if dialect.lower() == 'mysql': dialect = 'MySQL' if socketLoc != None: options['unix_socket'] = socketLoc elif dialect.lower() == 'oracle': dialect = 'Oracle' elif dialect.lower() == 'http': dialect = 'CouchDB' else: msg = "Unsupported dialect %s !" % dialect logging.error(msg) raise WMInitException(msg) myThread.dialect = dialect myThread.logger = logging myThread.dbFactory = DBFactory(logging, dbConfig, options) myThread.dbi = myThread.dbFactory.connect() # The transaction object will begin a transaction as soon as it is # initialized. I'd rather have the user handle that, so we'll commit # it here. myThread.transaction = Transaction(myThread.dbi) myThread.transaction.commit() return
def initInThread(self, parameters): """ Called when the thread is actually running in its own thread. Performs internal object setup. """ # Get the DB Factory we were passed by parent thread and assign to this # thread myThread = threading.currentThread() myThread.name = self.__class__.__name__ myThread.dbFactory = self.dbFactory # Now we're in our own thread, set the logger myThread.logger = self.logger (connectDialect, _junk) = self.component.config.CoreDatabase.connectUrl.split(":", 1) if connectDialect.lower() == "mysql": myThread.dialect = "MySQL" elif connectDialect.lower() == "oracle": myThread.dialect = "Oracle" logging.info("Initialising default database") myThread.dbi = myThread.dbFactory.connect() logging.info("Initialising default transaction") myThread.transaction = Transaction(myThread.dbi) self.setUpHeartbeat(myThread) self.setUpLogDB(myThread) # Call worker setup self.setup(parameters) myThread.transaction.commit()
def __init__(self, config = {}): """ __DatabasePage__ A page with a database connection (a WMCore.Database.DBFormatter) held in self.dbi. Look at the DBFormatter class for other handy helper methods, such as getBinds and formatDict. The DBFormatter class was originally intended to be extensively sub-classed, such that it's subclasses followed the DAO pattern. For web tools we do not generally do this, and you will normally access the database interface directly: binds = {'id': 123} sql = "select * from table where id = :id" result = self.dbi.processData(sql, binds) return self.formatDict(result) Although following the DAO pattern is still possible and encouraged where appropriate. However, if you want to use the DAO pattern it may be better to *not* expose the DAO classes and have a normal DatabasePage exposed that passes the database connection to all the DAO's. """ TemplatedPage.__init__(self, config) dbConfig = ConfigDBMap(config) conn = DBFactory(self, dbConfig.getDBUrl(), dbConfig.getOption()).connect() DBFormatter.__init__(self, self, conn) myThread = threading.currentThread() myThread.transaction = Transaction(conn) myThread.transaction.commit() return
def setUp(self): "make a logger instance and create tables" self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema() myThread = threading.currentThread() if myThread.dialect == 'MySQL': myThread.create = """ create table test (bind1 varchar(20), bind2 varchar(20)) ENGINE=InnoDB """ if myThread.dialect == 'SQLite': myThread.create = """ create table test (bind1 varchar(20), bind2 varchar(20))""" myThread.insert = """ insert into test (bind1, bind2) values (:bind1, :bind2) """ myThread.insert_binds = \ [ {'bind1':'value1a', 'bind2': 'value2a'},\ {'bind1':'value1b', 'bind2': 'value2b'},\ {'bind1':'value1c', 'bind2': 'value2d'} ] myThread.select = "select * from test" myThread = threading.currentThread() myThread.transaction = Transaction(myThread.dbi) myThread.transaction.processData(myThread.create) myThread.transaction.processData(myThread.insert, myThread.insert_binds) myThread.transaction.commit() return
def __init__(self, daoPackage, logger=None, dbi=None): """ ___init___ Initialize all the database connection attributes and the logging attritbutes. Create a DAO factory for given daoPackage as well. Finally, check to see if a transaction object has been created. If none exists, create one but leave the transaction closed. """ myThread = threading.currentThread() if logger: self.logger = logger else: self.logger = myThread.logger if dbi: self.dbi = dbi else: self.dbi = myThread.dbi self.daofactory = DAOFactory(package=daoPackage, logger=self.logger, dbinterface=self.dbi) if "transaction" not in dir(myThread): myThread.transaction = Transaction(self.dbi) return
def tearDown(self): """ Delete the databases """ myThread = threading.currentThread() myThread.transaction = Transaction(myThread.dbi) myThread.transaction.processData("drop table test") myThread.transaction.commit() self.testInit.clearDatabase()
def execute(self, siteName, conn=None, transaction=False): """ _execute_ Determine if the sites already exist in the dbsbuffer_location and attempt to lock the table using a "FOR UPDATE" parameters on the select statement. Insert any sites that do not already exist in the location table and return the IDs of all sites that were passed to this function. The sites will be returned as a dictionary where each key is the site name and the value is the site ID. This DAO will create it's own transaction and execute all SQL in that. This is done so that other transactions can pickup news sites and to avoid deadlocks. """ mySites = copy.deepcopy(siteName) nameMap = {} if type(mySites) == str: mySites = [mySites] myTransaction = Transaction(self.dbi) myTransaction.begin() binds = [] for location in mySites: binds.append({"location": location}) results = self.dbi.processData(self.existsSQL, binds, conn=myTransaction.conn, transaction=True) results = self.format(results) for result in results: nameMap[result[0]] = int(result[1]) mySites.remove(result[0]) binds = [] for location in mySites: binds.append({"location": location}) if len(binds) > 0: self.dbi.processData(self.sql, binds, conn=myTransaction.conn, transaction=True) results = self.dbi.processData(self.existsSQL, binds, conn=myTransaction.conn, transaction=True) results = self.format(results) for result in results: nameMap[result[0]] = int(result[1]) myTransaction.commit() return nameMap
def testListSitesTransaction(self): """ _testListSitesTransaction_ Verify that select behave appropriately when dealing with transactions. """ myThread = threading.currentThread() daoFactory = DAOFactory(package="WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) myThread.transaction.begin() localTransaction = Transaction(myThread.dbi) localTransaction.begin() locationNew = daoFactory(classname = "Locations.New") locationNew.execute("Satsuma", conn = myThread.transaction.conn, transaction = True) locationNew.execute("Choshu", conn = myThread.transaction.conn, transaction = True) listSites = daoFactory(classname = "Locations.ListSites") nonTransSites = listSites.execute(conn = localTransaction.conn, transaction = True) transSites = listSites.execute(conn = myThread.transaction.conn, transaction = True) assert len(nonTransSites) == 0, \ "Error: Wrong number of sites in non transaction list." assert len(transSites) == 2, \ "Error: Wrong number of sites in transaction list." assert "Satsuma" in transSites, \ "Error: Site missing in transaction list." assert "Choshu" in transSites, \ "Error: Site missing in transaction list." localTransaction.commit() myThread.transaction.commit() return
def execute(self, siteName, conn = None, transaction = False): if type(siteName) == str: binds = {"location": siteName} else: binds = [] for aLocation in siteName: binds.append({"location": aLocation}) myTransaction = Transaction(self.dbi) myTransaction.begin() nameMap = {} self.dbi.processData(self.sql, binds, conn = conn, transaction = transaction) results = self.dbi.processData(self.existsSQL, binds, conn = myTransaction.conn, transaction = True) results = self.format(results) for result in results: nameMap[result[0]] = int(result[1]) myTransaction.commit() return nameMap
def beginTransaction(self): """ _beginTransaction_ Begin a database transaction if one does not already exist. """ myThread = threading.currentThread() if "transaction" not in dir(myThread): myThread.transaction = Transaction(self.dbi) return False if myThread.transaction.transaction == None: myThread.transaction.begin() return False return True
def testD(self): raise nose.SkipTest config = self.testInit.getConfiguration() config.component_("TestComponent") config.TestComponent.logLevel = 'INFO' config.section_("General") self.tempDir = self.testInit.generateWorkDir(config) # try starting a component as a daemon: config.TestComponent.componentDir = os.path.join( \ self.tempDir, "Components/TestComponent2") os.makedirs(config.TestComponent.componentDir) testComponent = TestComponent(config) # we set the parent to true as we are testing testComponent.startDaemon(keepParent=True) time.sleep(2) daemonFile = os.path.join(config.TestComponent.componentDir, "Daemon.xml") details = Details(daemonFile) print('Is component alive: ' + str(details.isAlive())) # create msgService to send stop message. myThread = threading.currentThread() factory = WMFactory("msgService", "WMCore.MsgService." + \ myThread.dialect) myThread.transaction = Transaction(myThread.dbi) msgService = factory.loadObject("MsgService") msgService.registerAs("HarnessTest") myThread.transaction.commit() print( 'Publish a stop message to test if the component shutsdown gracefully' ) myThread.transaction.begin() msg = {'name': 'Stop', 'payload': ''} msgService.publish(msg) myThread.transaction.commit() msgService.finish() while details.isAlive(): print('Component has not received stop message') time.sleep(2) print('Daemon shutdown gracefully')
def initInThread(self): """ ThreadSlave (subclasses) are returned to a queue when no work is available and can be activated by a new thread when work is entered in the queue. This method is called when the ThreadSlave (subclass) is associated to a new thread. """ # we need to call this method only when it is called within a thread # otherwise these parameters are not accissible in the thread used # to call this threadslave. myThread = threading.currentThread() # we are now in our own thread, so we can pass the dbFactory reference # to here: myThread.dbFactory = self.dbFactory if self.component.config.CoreDatabase.dialect: myThread.dialect = self.component.config.CoreDatabase.dialect else: # FIXME we aren't using the DIALECT environment variable anymore myThread.dialect = os.getenv("DIALECT") #TODO: remove as much as possible logging statements or make them debug myThread.logger = logging.getLogger() logging.info("THREAD: Initializing default database") logging.info("THREAD: Check if connection is through socket") logging.info("THREAD: Building database connection string") # we ensured that we use the dbFactory object from our parent # thread so we have only one engine in the application. myThread.dbi = myThread.dbFactory.connect() logging.info("THREAD: Initializing default transaction") myThread.transaction = Transaction(myThread.dbi) logging.info("THREAD: Loading backend") logging.info("THREAD constructor finished")
def testListSitesTransaction(self): """ _testListSitesTransaction_ Verify that select behave appropriately when dealing with transactions. """ myThread = threading.currentThread() daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) myThread.transaction.begin() localTransaction = Transaction(myThread.dbi) localTransaction.begin() locationNew = daoFactory(classname="Locations.New") locationNew.execute("Satsuma", conn=myThread.transaction.conn, transaction=True) locationNew.execute("Choshu", conn=myThread.transaction.conn, transaction=True) listSites = daoFactory(classname="Locations.ListSites") nonTransSites = listSites.execute(conn=localTransaction.conn, transaction=True) transSites = listSites.execute(conn=myThread.transaction.conn, transaction=True) assert len(nonTransSites) == 0, \ "Error: Wrong number of sites in non transaction list." assert len(transSites) == 2, \ "Error: Wrong number of sites in transaction list." assert "Satsuma" in transSites, \ "Error: Site missing in transaction list." assert "Choshu" in transSites, \ "Error: Site missing in transaction list." localTransaction.commit() myThread.transaction.commit() return
def initInThread(self): """ Default intialization of the harness including setting some diagnostic messages. This method is called when we call 'prepareToStart' """ try: self.messages = {} compName = self.config.Agent.componentName compSect = getattr(self.config, compName, None) if not hasattr(compSect, "logFile"): if not getattr(compSect, 'componentDir', None): errorMessage = "No componentDir for log entries found!\n" errorMessage += "Harness cannot run without componentDir.\n" logging.error(errorMessage) raise HarnessException(errorMessage) compSect.logFile = os.path.join(compSect.componentDir, "ComponentLog") print('Log file is: ' + compSect.logFile) logHandler = RotatingFileHandler(compSect.logFile, "a", 1000000000, 3) logMsgFormat = getattr( compSect, "logMsgFormat", "%(asctime)s:%(thread)d:%(levelname)s:%(module)s:%(message)s") logFormatter = \ logging.Formatter(logMsgFormat) logHandler.setFormatter(logFormatter) logLevelName = getattr(compSect, 'logLevel', 'INFO') logLevel = getattr(logging, logLevelName) logging.getLogger().addHandler(logHandler) logging.getLogger().setLevel(logLevel) self.logMsg = { 'DEBUG': logging.DEBUG, 'ERROR': logging.ERROR, 'NOTSET': logging.NOTSET, 'CRITICAL': logging.CRITICAL, 'WARNING': logging.WARNING, 'INFO': logging.INFO, 'SQLDEBUG': logging.SQLDEBUG } if hasattr(compSect, "logLevel") and compSect.logLevel in self.logMsg: logging.getLogger().setLevel(self.logMsg[compSect.logLevel]) WMLogging.sqldebug("wmcore level debug:") # If not previously set, force wmcore cache to current path if not os.environ.get('WMCORE_CACHE_DIR'): os.environ['WMCORE_CACHE_DIR'] = os.path.join( compSect.componentDir, '.wmcore_cache') logging.info(">>>Starting: " + compName + '<<<') # check which backend to use: MySQL, Oracle, etc... for core # services. # we recognize there can be more than one database. # be we offer a default database that is used for core services. logging.info(">>>Initializing default database") logging.info(">>>Check if connection is through socket") myThread = threading.currentThread() myThread.logger = logging.getLogger() logging.info(">>>Setting config for thread: ") myThread.config = self.config logging.info(">>>Building database connection string") # check if there is a premade string if not build it yourself. dbConfig = ConfigDBMap(self.config) dbStr = dbConfig.getDBUrl() options = dbConfig.getOption() # we only want one DBFactory per database so we will need to # to pass this on in case we are using threads. myThread.dbFactory = DBFactory(myThread.logger, dbStr, options) myThread.sql_transaction = True if myThread.dbFactory.engine: myThread.dbi = myThread.dbFactory.connect() myThread.transaction = Transaction(myThread.dbi) else: myThread.dbi = myThread.config.CoreDatabase.connectUrl myThread.sql_transaction = False # Attach a worker manager object to the main thread if not hasattr(myThread, 'workerThreadManager'): myThread.workerThreadManager = WorkerThreadManager(self) else: myThread.workerThreadManager.terminateSlaves.clear() myThread.workerThreadManager.pauseWorkers() logging.info(">>>Initialize transaction dictionary") (connectDialect, dummy) = dbStr.split(":", 1) if connectDialect.lower() == 'mysql': myThread.dialect = 'MySQL' elif connectDialect.lower() == 'oracle': myThread.dialect = 'Oracle' logging.info("Harness part constructor finished") except Exception as ex: logging.critical("Problem instantiating " + str(ex)) logging.error("Traceback: %s", str(traceback.format_exc())) raise