Esempio n. 1
0
    def execute(self, siteName, conn = None, transaction = False):
        """
        _execute_

        Determine if the sites already exist in the dbsbuffer_location and
        insert any sites that do not already exist in the location table and
        return the IDs of all sites that were passed to this function.

        The sites will be returned as a dictionary where each key is the site
        name and the value is the site ID.

        This DAO will create it's own transaction and execute all SQL in that.
        This is done so that other transactions can pickup news sites and to
        avoid deadlocks.
        """
        mySites = copy.deepcopy(siteName)
        nameMap = {}

        if type(mySites) == str:
            mySites = [mySites]

        myTransaction = Transaction(self.dbi)
        myTransaction.begin()

        binds = []
        for location in mySites:
            binds.append({"location": location})

        results = self.dbi.processData(self.existsSQL, binds,
                                       conn = myTransaction.conn,
                                       transaction = True)
        results = self.format(results)
        for result in results:
            nameMap[result[0]] = int(result[1])
            mySites.remove(result[0])

        binds = []
        for location in mySites:
            binds.append({"location": location})

        if len(binds) > 0:
            try:
                self.dbi.processData(self.sql, binds, conn = myTransaction.conn,
                                     transaction = True)
            except Exception, ex:
                if "orig" in dir(ex) and type(ex.orig) != tuple:
                    if str(ex.orig).find("ORA-00001: unique constraint") != -1 and \
                       str(ex.orig).find("DBSBUFFER_LOCATION_UNIQUE") != -1:
                        return
                raise ex

            results = self.dbi.processData(self.existsSQL, binds,
                                           conn = myTransaction.conn,
                                           transaction = True)

            results = self.format(results)
            for result in results:
                nameMap[result[0]] = int(result[1])
Esempio n. 2
0
    def execute(self, siteName, conn = None, transaction = False):
        """
        _execute_

        Determine if the sites already exist in the dbsbuffer_location and
        attempt to lock the table using a "FOR UPDATE" parameters on the
        select statement.  Insert any sites that do not already exist in the
        location table and return the IDs of all sites that were passed to this
        function.

        The sites will be returned as a dictionary where each key is the site
        name and the value is the site ID.

        This DAO will create it's own transaction and execute all SQL in that.
        This is done so that other transactions can pickup news sites and to
        avoid deadlocks.
        """
        mySites = copy.deepcopy(siteName)
        nameMap = {}

        if type(mySites) == str:
            mySites = [mySites]

        myTransaction = Transaction(self.dbi)
        myTransaction.begin()

        binds = []
        for location in mySites:
            binds.append({"location": location})

        results = self.dbi.processData(self.existsSQL, binds,
                                       conn = myTransaction.conn,
                                       transaction = True)
        results = self.format(results)
        for result in results:
            nameMap[result[0]] = int(result[1])
            mySites.remove(result[0])

        binds = []
        for location in mySites:
            binds.append({"location": location})

        if len(binds) > 0:
            self.dbi.processData(self.sql, binds, conn = myTransaction.conn,
                                 transaction = True)
            results = self.dbi.processData(self.existsSQL, binds,
                                           conn = myTransaction.conn,
                                           transaction = True)

            results = self.format(results)
            for result in results:
                nameMap[result[0]] = int(result[1])

        myTransaction.commit()
        return nameMap
Esempio n. 3
0
    def setDatabaseConnection(self, dbConfig, dialect, socketLoc=None):
        """
        Sets the default connection parameters, without having to worry
        much on what attributes need to be set. This is esepcially
        advantagous for developers of third party projects that want
        to use only parts of the WMCore lib.

        The class differentiates between different formats used by external
        projects. External project formats that are supported can activated
        it by setting the flavor flag.
        """
        myThread = threading.currentThread()
        if getattr(myThread, "dialect", None) != None:
            # Database is already initialized, we'll create a new
            # transaction and move on.
            if hasattr(myThread, "transaction"):
                if myThread.transaction != None:
                    myThread.transaction.commit()

            myThread.transaction = Transaction(myThread.dbi)
            return

        options = {}
        if dialect.lower() == 'mysql':
            dialect = 'MySQL'
            if socketLoc != None:
                options['unix_socket'] = socketLoc
        elif dialect.lower() == 'oracle':
            dialect = 'Oracle'
        elif dialect.lower() == 'http':
            dialect = 'CouchDB'
        else:
            msg = "Unsupported dialect %s !" % dialect
            logging.error(msg)
            raise WMInitException(msg)

        myThread.dialect = dialect
        myThread.logger = logging
        myThread.dbFactory = DBFactory(logging, dbConfig, options)
        myThread.dbi = myThread.dbFactory.connect()

        # The transaction object will begin a transaction as soon as it is
        # initialized.  I'd rather have the user handle that, so we'll commit
        # it here.
        myThread.transaction = Transaction(myThread.dbi)
        myThread.transaction.commit()

        return
Esempio n. 4
0
    def initInThread(self, parameters):
        """
        Called when the thread is actually running in its own thread. Performs
        internal object setup.
        """
        # Get the DB Factory we were passed by parent thread and assign to this
        # thread
        myThread = threading.currentThread()
        myThread.name = self.__class__.__name__
        myThread.dbFactory = self.dbFactory

        # Now we're in our own thread, set the logger
        myThread.logger = self.logger

        (connectDialect, _junk) = self.component.config.CoreDatabase.connectUrl.split(":", 1)

        if connectDialect.lower() == "mysql":
            myThread.dialect = "MySQL"
        elif connectDialect.lower() == "oracle":
            myThread.dialect = "Oracle"

        logging.info("Initialising default database")
        myThread.dbi = myThread.dbFactory.connect()
        logging.info("Initialising default transaction")
        myThread.transaction = Transaction(myThread.dbi)

        self.setUpHeartbeat(myThread)
        self.setUpLogDB(myThread)

        # Call worker setup
        self.setup(parameters)
        myThread.transaction.commit()
Esempio n. 5
0
    def __init__(self, config = {}):
        """
        __DatabasePage__

        A page with a database connection (a WMCore.Database.DBFormatter) held
        in self.dbi. Look at the DBFormatter class for other handy helper
        methods, such as getBinds and formatDict.

        The DBFormatter class was originally intended to be extensively
        sub-classed, such that it's subclasses followed the DAO pattern. For web
        tools we do not generally do this, and you will normally access the
        database interface directly:

        binds = {'id': 123}
        sql = "select * from table where id = :id"
        result = self.dbi.processData(sql, binds)
        return self.formatDict(result)

        Although following the DAO pattern is still possible and encouraged
        where appropriate. However, if you want to use the DAO pattern it may be
        better to *not* expose the DAO classes and have a normal DatabasePage
        exposed that passes the database connection to all the DAO's.
        """
        TemplatedPage.__init__(self, config)
        dbConfig = ConfigDBMap(config)
        conn = DBFactory(self, dbConfig.getDBUrl(), dbConfig.getOption()).connect()
        DBFormatter.__init__(self, self, conn)
        myThread = threading.currentThread()
        myThread.transaction = Transaction(conn)
        myThread.transaction.commit()
        return
Esempio n. 6
0
    def setUp(self):
        "make a logger instance and create tables"

        self.testInit = TestInit(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setSchema()

        myThread = threading.currentThread()
        if myThread.dialect == 'MySQL':
            myThread.create = """
create table test (bind1 varchar(20), bind2 varchar(20)) ENGINE=InnoDB """
        if myThread.dialect == 'SQLite':
            myThread.create = """
                create table test (bind1 varchar(20), bind2 varchar(20))"""

        myThread.insert = """
insert into test (bind1, bind2) values (:bind1, :bind2) """
        myThread.insert_binds = \
          [ {'bind1':'value1a', 'bind2': 'value2a'},\
            {'bind1':'value1b', 'bind2': 'value2b'},\
            {'bind1':'value1c', 'bind2': 'value2d'} ]
        myThread.select = "select * from test"

        myThread = threading.currentThread()
        myThread.transaction = Transaction(myThread.dbi)
        myThread.transaction.processData(myThread.create)
        myThread.transaction.processData(myThread.insert,
                                         myThread.insert_binds)
        myThread.transaction.commit()

        return
Esempio n. 7
0
    def __init__(self, daoPackage, logger=None, dbi=None):
        """
        ___init___

        Initialize all the database connection attributes and the logging
        attritbutes.  Create a DAO factory for given daoPackage as well. Finally,
        check to see if a transaction object has been created.  If none exists,
        create one but leave the transaction closed.
        """
        myThread = threading.currentThread()
        if logger:
            self.logger = logger
        else:
            self.logger = myThread.logger
        if dbi:
            self.dbi = dbi
        else:
            self.dbi = myThread.dbi

        self.daofactory = DAOFactory(package=daoPackage,
                                     logger=self.logger,
                                     dbinterface=self.dbi)

        if "transaction" not in dir(myThread):
            myThread.transaction = Transaction(self.dbi)

        return
Esempio n. 8
0
 def tearDown(self):
     """
     Delete the databases
     """
     myThread = threading.currentThread()
     myThread.transaction = Transaction(myThread.dbi)
     myThread.transaction.processData("drop table test")
     myThread.transaction.commit()
     self.testInit.clearDatabase()
Esempio n. 9
0
    def execute(self, siteName, conn=None, transaction=False):
        """
        _execute_

        Determine if the sites already exist in the dbsbuffer_location and
        attempt to lock the table using a "FOR UPDATE" parameters on the
        select statement.  Insert any sites that do not already exist in the
        location table and return the IDs of all sites that were passed to this
        function.

        The sites will be returned as a dictionary where each key is the site
        name and the value is the site ID.

        This DAO will create it's own transaction and execute all SQL in that.
        This is done so that other transactions can pickup news sites and to
        avoid deadlocks.
        """
        mySites = copy.deepcopy(siteName)
        nameMap = {}

        if type(mySites) == str:
            mySites = [mySites]

        myTransaction = Transaction(self.dbi)
        myTransaction.begin()

        binds = []
        for location in mySites:
            binds.append({"location": location})

        results = self.dbi.processData(self.existsSQL,
                                       binds,
                                       conn=myTransaction.conn,
                                       transaction=True)
        results = self.format(results)
        for result in results:
            nameMap[result[0]] = int(result[1])
            mySites.remove(result[0])

        binds = []
        for location in mySites:
            binds.append({"location": location})

        if len(binds) > 0:
            self.dbi.processData(self.sql,
                                 binds,
                                 conn=myTransaction.conn,
                                 transaction=True)
            results = self.dbi.processData(self.existsSQL,
                                           binds,
                                           conn=myTransaction.conn,
                                           transaction=True)

            results = self.format(results)
            for result in results:
                nameMap[result[0]] = int(result[1])

        myTransaction.commit()
        return nameMap
Esempio n. 10
0
    def testListSitesTransaction(self):
        """
        _testListSitesTransaction_

        Verify that select behave appropriately when dealing with transactions.
        """
        myThread = threading.currentThread()
        daoFactory = DAOFactory(package="WMCore.WMBS", logger = myThread.logger,
                                dbinterface = myThread.dbi)
        
        myThread.transaction.begin()

        localTransaction = Transaction(myThread.dbi)
        localTransaction.begin()
        
        locationNew = daoFactory(classname = "Locations.New")

        locationNew.execute("Satsuma", conn = myThread.transaction.conn, transaction = True)
        locationNew.execute("Choshu", conn = myThread.transaction.conn, transaction = True)

        listSites = daoFactory(classname = "Locations.ListSites")
        nonTransSites = listSites.execute(conn = localTransaction.conn, transaction = True)
        transSites = listSites.execute(conn = myThread.transaction.conn, transaction = True)

        assert len(nonTransSites) == 0, \
               "Error: Wrong number of sites in non transaction list."
        assert len(transSites) == 2, \
               "Error: Wrong number of sites in transaction list."
        assert "Satsuma" in transSites, \
               "Error: Site missing in transaction list."
        assert "Choshu" in transSites, \
               "Error: Site missing in transaction list."

        localTransaction.commit()
        myThread.transaction.commit()
        return
Esempio n. 11
0
    def execute(self, siteName, conn = None, transaction = False):

        if type(siteName) == str:
            binds = {"location": siteName}
        else:
            binds = []
            for aLocation in siteName:
                binds.append({"location": aLocation})

        myTransaction = Transaction(self.dbi)
        myTransaction.begin()

        nameMap = {}
        self.dbi.processData(self.sql, binds, conn = conn, 
                             transaction = transaction)
        results = self.dbi.processData(self.existsSQL, binds,
                                           conn = myTransaction.conn,
                                           transaction = True)
        results = self.format(results)
        for result in results:
            nameMap[result[0]] = int(result[1])

        myTransaction.commit()
        return nameMap
Esempio n. 12
0
    def beginTransaction(self):
        """
        _beginTransaction_

        Begin a database transaction if one does not already exist.
        """
        myThread = threading.currentThread()

        if "transaction" not in dir(myThread):
            myThread.transaction = Transaction(self.dbi)
            return False

        if myThread.transaction.transaction == None:
            myThread.transaction.begin()
            return False

        return True
Esempio n. 13
0
    def testD(self):
        raise nose.SkipTest
        config = self.testInit.getConfiguration()
        config.component_("TestComponent")
        config.TestComponent.logLevel = 'INFO'
        config.section_("General")
        self.tempDir = self.testInit.generateWorkDir(config)
        # try starting a component as a daemon:
        config.TestComponent.componentDir = os.path.join( \
            self.tempDir, "Components/TestComponent2")
        os.makedirs(config.TestComponent.componentDir)
        testComponent = TestComponent(config)
        # we set the parent to true as we are testing
        testComponent.startDaemon(keepParent=True)
        time.sleep(2)
        daemonFile = os.path.join(config.TestComponent.componentDir,
                                  "Daemon.xml")
        details = Details(daemonFile)
        print('Is component alive: ' + str(details.isAlive()))

        # create msgService to send stop message.
        myThread = threading.currentThread()
        factory = WMFactory("msgService", "WMCore.MsgService." + \
                            myThread.dialect)
        myThread.transaction = Transaction(myThread.dbi)
        msgService = factory.loadObject("MsgService")
        msgService.registerAs("HarnessTest")
        myThread.transaction.commit()

        print(
            'Publish a stop message to test if the component shutsdown gracefully'
        )
        myThread.transaction.begin()
        msg = {'name': 'Stop', 'payload': ''}
        msgService.publish(msg)
        myThread.transaction.commit()

        msgService.finish()

        while details.isAlive():
            print('Component has not received stop message')
            time.sleep(2)
        print('Daemon shutdown gracefully')
Esempio n. 14
0
    def initInThread(self):
        """
        ThreadSlave (subclasses) are returned to a queue when no work
        is available and can be activated by a new thread when work
        is entered in the queue. This method is called when the ThreadSlave
        (subclass) is associated to a new thread.
        """

        # we need to call this method only when it is called within a thread
        # otherwise these parameters are not accissible in the thread used
        # to call this threadslave.

        myThread = threading.currentThread()
        # we are now in our own thread, so we can pass the dbFactory reference
        # to here:
        myThread.dbFactory = self.dbFactory

        if self.component.config.CoreDatabase.dialect:
            myThread.dialect = self.component.config.CoreDatabase.dialect
        else:
            # FIXME we aren't using the DIALECT environment variable anymore
            myThread.dialect = os.getenv("DIALECT")

        #TODO: remove as much as possible logging statements or make them debug
        myThread.logger = logging.getLogger()

        logging.info("THREAD: Initializing default database")
        logging.info("THREAD: Check if connection is through socket")
        logging.info("THREAD: Building database connection string")
        # we ensured that we use the dbFactory object from our parent
        # thread so we have only one engine in the application.
        myThread.dbi = myThread.dbFactory.connect()
        logging.info("THREAD: Initializing default transaction")
        myThread.transaction = Transaction(myThread.dbi)
        logging.info("THREAD: Loading backend")
        logging.info("THREAD constructor finished")
Esempio n. 15
0
    def testListSitesTransaction(self):
        """
        _testListSitesTransaction_

        Verify that select behave appropriately when dealing with transactions.
        """
        myThread = threading.currentThread()
        daoFactory = DAOFactory(package="WMCore.WMBS",
                                logger=myThread.logger,
                                dbinterface=myThread.dbi)

        myThread.transaction.begin()

        localTransaction = Transaction(myThread.dbi)
        localTransaction.begin()

        locationNew = daoFactory(classname="Locations.New")

        locationNew.execute("Satsuma",
                            conn=myThread.transaction.conn,
                            transaction=True)
        locationNew.execute("Choshu",
                            conn=myThread.transaction.conn,
                            transaction=True)

        listSites = daoFactory(classname="Locations.ListSites")
        nonTransSites = listSites.execute(conn=localTransaction.conn,
                                          transaction=True)
        transSites = listSites.execute(conn=myThread.transaction.conn,
                                       transaction=True)

        assert len(nonTransSites) == 0, \
               "Error: Wrong number of sites in non transaction list."
        assert len(transSites) == 2, \
               "Error: Wrong number of sites in transaction list."
        assert "Satsuma" in transSites, \
               "Error: Site missing in transaction list."
        assert "Choshu" in transSites, \
               "Error: Site missing in transaction list."

        localTransaction.commit()
        myThread.transaction.commit()
        return
Esempio n. 16
0
    def initInThread(self):
        """
        Default intialization of the harness including setting some diagnostic
        messages. This method is called when we call 'prepareToStart'
        """
        try:
            self.messages = {}

            compName = self.config.Agent.componentName
            compSect = getattr(self.config, compName, None)
            if not hasattr(compSect, "logFile"):
                if not getattr(compSect, 'componentDir', None):
                    errorMessage = "No componentDir for log entries found!\n"
                    errorMessage += "Harness cannot run without componentDir.\n"
                    logging.error(errorMessage)
                    raise HarnessException(errorMessage)
                compSect.logFile = os.path.join(compSect.componentDir,
                                                "ComponentLog")
            print('Log file is: ' + compSect.logFile)
            logHandler = RotatingFileHandler(compSect.logFile, "a", 1000000000,
                                             3)
            logMsgFormat = getattr(
                compSect, "logMsgFormat",
                "%(asctime)s:%(thread)d:%(levelname)s:%(module)s:%(message)s")
            logFormatter = \
                logging.Formatter(logMsgFormat)
            logHandler.setFormatter(logFormatter)
            logLevelName = getattr(compSect, 'logLevel', 'INFO')
            logLevel = getattr(logging, logLevelName)
            logging.getLogger().addHandler(logHandler)
            logging.getLogger().setLevel(logLevel)
            self.logMsg = {
                'DEBUG': logging.DEBUG,
                'ERROR': logging.ERROR,
                'NOTSET': logging.NOTSET,
                'CRITICAL': logging.CRITICAL,
                'WARNING': logging.WARNING,
                'INFO': logging.INFO,
                'SQLDEBUG': logging.SQLDEBUG
            }
            if hasattr(compSect,
                       "logLevel") and compSect.logLevel in self.logMsg:
                logging.getLogger().setLevel(self.logMsg[compSect.logLevel])
            WMLogging.sqldebug("wmcore level debug:")

            # If not previously set, force wmcore cache to current path
            if not os.environ.get('WMCORE_CACHE_DIR'):
                os.environ['WMCORE_CACHE_DIR'] = os.path.join(
                    compSect.componentDir, '.wmcore_cache')

            logging.info(">>>Starting: " + compName + '<<<')
            # check which backend to use: MySQL, Oracle, etc... for core
            # services.
            # we recognize there can be more than one database.
            # be we offer a default database that is used for core services.
            logging.info(">>>Initializing default database")
            logging.info(">>>Check if connection is through socket")
            myThread = threading.currentThread()
            myThread.logger = logging.getLogger()
            logging.info(">>>Setting config for thread: ")
            myThread.config = self.config

            logging.info(">>>Building database connection string")
            # check if there is a premade string if not build it yourself.
            dbConfig = ConfigDBMap(self.config)
            dbStr = dbConfig.getDBUrl()
            options = dbConfig.getOption()
            # we only want one DBFactory per database so we will need to
            # to pass this on in case we are using threads.
            myThread.dbFactory = DBFactory(myThread.logger, dbStr, options)

            myThread.sql_transaction = True
            if myThread.dbFactory.engine:

                myThread.dbi = myThread.dbFactory.connect()
                myThread.transaction = Transaction(myThread.dbi)

            else:

                myThread.dbi = myThread.config.CoreDatabase.connectUrl
                myThread.sql_transaction = False

            # Attach a worker manager object to the main thread
            if not hasattr(myThread, 'workerThreadManager'):
                myThread.workerThreadManager = WorkerThreadManager(self)
            else:
                myThread.workerThreadManager.terminateSlaves.clear()
            myThread.workerThreadManager.pauseWorkers()

            logging.info(">>>Initialize transaction dictionary")

            (connectDialect, dummy) = dbStr.split(":", 1)

            if connectDialect.lower() == 'mysql':
                myThread.dialect = 'MySQL'
            elif connectDialect.lower() == 'oracle':
                myThread.dialect = 'Oracle'

            logging.info("Harness part constructor finished")
        except Exception as ex:
            logging.critical("Problem instantiating " + str(ex))
            logging.error("Traceback: %s", str(traceback.format_exc()))
            raise