Esempio n. 1
0
    def __init__(self, config):
        """
        Initialise class members
        """
        #Need a better way to test this without turning off this next line
        BaseDaemon.__init__(self, config, 'AsyncTransfer')

        self.dropbox_dir = '%s/dropbox/outputs' % self.config.componentDir
        if not os.path.isdir(self.dropbox_dir):
            try:
                os.makedirs(self.dropbox_dir)
            except OSError as e:
                if e.errno == errno.EEXIST:
                    pass
                else:
                    self.logger.error('Unknown error in mkdir' % e.errno)
                    raise
        server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
        self.db = server.connectDatabase(self.config.files_database)
        config_server = CouchServer(dburl=self.config.config_couch_instance)
        self.config_db = config_server.connectDatabase(self.config.config_database)
        self.logger.debug('Connected to CouchDB')
        self.pool = Pool(processes=self.config.pool_size)
        try:
            self.phedex = PhEDEx(responseType='xml', dict = {'key': self.config.opsProxy, 'cert': self.config.opsProxy})
        except Exception as e:
            self.logger.exception('PhEDEx exception: %s' % e)
        # Set up a factory for loading plugins
        self.factory = WMFactory(self.config.schedAlgoDir, namespace = self.config.schedAlgoDir)

        result_list = []
        current_running = []
Esempio n. 2
0
class Algo:
    """
    Plugins parent class.
    """
    def __init__(self, config, logger, users, pool_size):
        """
        Initialise class members
        """
        self.config = config
        self.logger = logger
        self.asyncServer = CouchServer(self.config.couch_instance, ckey = self.config.opsProxy, cert = self.config.opsProxy)
        self.db = self.asyncServer.connectDatabase(self.config.files_database)
        self.config_db = self.asyncServer.connectDatabase(self.config.config_database)
        self.users = users
        self.pool_size = pool_size

    def __call__(self):
        """
        __call__ should be over written by subclasses such that useful results are returned
        """
        return []

    def updateSource(self, inputDict):
        """
        UpdateSource should be over written by subclasses to make a specific update in the source
        """
        return []
Esempio n. 3
0
    def __init__(self, config):
        BaseWorkerThread.__init__(self)
        self.config = config.Statistics

        try:
            self.logger.setLevel(self.config.log_level)
        except:
            import logging
            self.logger = logging.getLogger()
            self.logger.setLevel(self.config.log_level)

        self.logger.debug('Configuration loaded')

        server = CouchServer(
            dburl=self.config.couch_instance,
            ckey=self.config.opsProxy,
            cert=self.config.opsProxy)
        self.db = server.connectDatabase(self.config.files_database)
        config_server = CouchServer(
            dburl=self.config.config_couch_instance,
            ckey=self.config.opsProxy,
            cert=self.config.opsProxy)
        self.config_db = config_server.connectDatabase(
            self.config.config_database)
        self.logger.debug('Connected to CouchDB')

        statserver = CouchServer(self.config.couch_statinstance)
        self.statdb = statserver.connectDatabase(
            self.config.statitics_database)
        self.logger.debug('Connected to Stat CouchDB')

        self.iteration_docs = []
        self.exptime = None
Esempio n. 4
0
    def __init__(self, config):
        BaseWorkerThread.__init__(self)
        self.config = config.Statistics

        try:
            self.logger.setLevel(self.config.log_level)
        except:
            import logging
            self.logger = logging.getLogger()
            self.logger.setLevel(self.config.log_level)

        self.logger.debug('Configuration loaded')

        server = CouchServer(dburl=self.config.couch_instance,
                             ckey=self.config.opsProxy,
                             cert=self.config.opsProxy)
        self.db = server.connectDatabase(self.config.files_database)

        config_server = CouchServer(dburl=self.config.config_couch_instance,
                                    ckey=self.config.opsProxy,
                                    cert=self.config.opsProxy)
        self.config_db = config_server.connectDatabase(
            self.config.config_database)
        self.mon_db = server.connectDatabase(self.config.mon_database)
        self.logger.debug('Connected to CouchDB')

        statserver = CouchServer(self.config.couch_statinstance)
        self.statdb = statserver.connectDatabase(
            self.config.statitics_database)
        self.logger.debug('Connected to Stat CouchDB')

        self.iteration_docs = []
        self.exptime = None
Esempio n. 5
0
    def __init__(self, config):
        """
        Initialise class members
        """
        # Need a better way to test this without turning off this next line
        BaseWorkerThread.__init__(self)
        # logging.basicConfig(format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s',datefmt = '%m-%d %H:%M')
        # self.logger = logging.getLogger()
        # self.logger is set up by the BaseWorkerThread, we just set it's level

        self.config = config.AsyncTransfer
        try:
            self.logger.setLevel(self.config.log_level)
        except:
            import logging

            self.logger = logging.getLogger()
            self.logger.setLevel(self.config.log_level)

        self.logger.debug("Configuration loaded")
        server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
        self.db = server.connectDatabase(self.config.files_database)
        config_server = CouchServer(dburl=self.config.config_couch_instance)
        self.config_db = config_server.connectDatabase(self.config.config_database)
        self.logger.debug("Connected to CouchDB")
        self.pool = Pool(processes=self.config.pool_size)
        try:
            self.phedex = PhEDEx(responseType="xml")
        except Exception, e:
            self.logger.exception("PhEDEx exception: %s" % e)
Esempio n. 6
0
    def __init__(self, config):
        """
        Initialise class members
        """
        #Need a better way to test this without turning off this next line
        BaseWorkerThread.__init__(self)
        #logging.basicConfig(format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s',datefmt = '%m-%d %H:%M')
        #self.logger = logging.getLogger()
        # self.logger is set up by the BaseWorkerThread, we just set it's level

        self.config = config.AsyncTransfer
        try:
            self.logger.setLevel(self.config.log_level)
        except:
            import logging
            self.logger = logging.getLogger()
            self.logger.setLevel(self.config.log_level)

        self.logger.debug('Configuration loaded')
        server = CouchServer(dburl=self.config.couch_instance,
                             ckey=self.config.opsProxy,
                             cert=self.config.opsProxy)
        self.db = server.connectDatabase(self.config.files_database)
        config_server = CouchServer(dburl=self.config.config_couch_instance)
        self.config_db = config_server.connectDatabase(
            self.config.config_database)
        self.logger.debug('Connected to CouchDB')
        self.pool = Pool(processes=self.config.pool_size)
        try:
            self.phedex = PhEDEx(responseType='xml')
        except Exception, e:
            self.logger.exception('PhEDEx exception: %s' % e)
Esempio n. 7
0
class Algo:
    """
    Plugins parent class.
    """
    def __init__(self, config, logger, users, pool_size):
        """
        Initialise class members
        """
        self.config = config
        self.logger = logger
        self.asyncServer = CouchServer(self.config.couch_instance)
        self.db = self.asyncServer.connectDatabase(self.config.files_database)
        self.config_db = self.asyncServer.connectDatabase(self.config.config_database)
        self.users = users
        self.pool_size = pool_size

    def __call__(self):
        """
        __call__ should be over written by subclasses such that useful results are returned
        """
        return []

    def updateSource(self, inputDict):
        """
        UpdateSource should be over written by subclasses to make a specific update in the source
        """
        return []
Esempio n. 8
0
class ChangeState(WMObject, WMConnectionBase):
    """
    Propagate the state of a job through the JSM.
    """
    def __init__(self, config, couchDbName=None):
        WMObject.__init__(self, config)
        WMConnectionBase.__init__(self, "WMCore.WMBS")

        if couchDbName == None:
            self.dbname = getattr(self.config.JobStateMachine, "couchDBName")
        else:
            self.dbname = couchDbName

        try:
            self.couchdb = CouchServer(self.config.JobStateMachine.couchurl)
            self.jobsdatabase = self.couchdb.connectDatabase("%s/jobs" %
                                                             self.dbname,
                                                             size=250)
            self.fwjrdatabase = self.couchdb.connectDatabase("%s/fwjrs" %
                                                             self.dbname,
                                                             size=250)
            self.jsumdatabase = self.couchdb.connectDatabase(getattr(
                self.config.JobStateMachine, 'jobSummaryDBName'),
                                                             size=250)
        except Exception, ex:
            logging.error("Error connecting to couch: %s" % str(ex))
            self.jobsdatabase = None
            self.fwjrdatabase = None
            self.jsumdatabase = None

        try:
            self.dashboardReporter = DashboardReporter(config)
        except Exception, ex:
            logging.error("Error setting up the \
                          dashboard reporter: %s" % str(ex))
Esempio n. 9
0
    def testB_testErrors(self):
        """
        _testErrors_

        Test with a failed FWJR
        """

        myThread = threading.currentThread()

        config = self.getConfig()
        workloadPath = os.path.join(self.testDir, 'specDir', 'spec.pkl')
        workload     = self.createWorkload(workloadName = workloadPath)
        testJobGroup = self.createTestJobGroup(config = config,
                                               name = workload.name(),
                                               specLocation = workloadPath,
                                               error = True)

        cachePath = os.path.join(config.JobCreator.jobCacheDir,
                                 "TestWorkload", "ReReco")
        os.makedirs(cachePath)
        self.assertTrue(os.path.exists(cachePath))

        couchdb      = CouchServer(config.JobStateMachine.couchurl)
        jobdb        = couchdb.connectDatabase("%s/jobs" % self.databaseName)
        fwjrdb       = couchdb.connectDatabase("%s/fwjrs" % self.databaseName)
        jobdb.loadView("JobDump", "jobsByWorkflowName",
                        options = {"startkey": [workload.name()],
                                   "endkey": [workload.name(), {}]})['rows']
        fwjrdb.loadView("FWJRDump", "fwjrsByWorkflowName",
                        options = {"startkey": [workload.name()],
                                   "endkey": [workload.name(), {}]})['rows']

        testTaskArchiver = TaskArchiverPoller(config = config)
        testTaskArchiver.algorithm()

        dbname       = getattr(config.JobStateMachine, "couchDBName")
        workdatabase = couchdb.connectDatabase("%s/workloadsummary" % dbname)

        workloadSummary = workdatabase.document(id = workload.name())

        self.assertEqual(workloadSummary['errors']['/TestWorkload/ReReco']['failureTime'], 500)
        self.assertTrue(workloadSummary['errors']['/TestWorkload/ReReco']['cmsRun1'].has_key('99999'))

        failedRunInfo = workloadSummary['errors']['/TestWorkload/ReReco']['cmsRun1']['99999']['runs']
        for key, value in failedRunInfo.items():
            failedRunInfo[key] = list(set(value))
        self.assertEquals(failedRunInfo, {'10' : [12312]},
                          "Wrong lumi information in the summary for failed jobs")

        # Check the failures by site histograms
        self.assertEqual(workloadSummary['histograms']['workflowLevel']['failuresBySite']['data']['T1_IT_CNAF']['Failed Jobs'], 10)
        self.assertEqual(workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']['cmsRun1']['errorsBySite']['data']['T1_IT_CNAF']['99999'], 10)
        self.assertEqual(workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']['cmsRun1']['errorsBySite']['data']['T1_IT_CNAF']['8020'], 10)
        self.assertEqual(workloadSummary['histograms']['workflowLevel']['failuresBySite']['average']['Failed Jobs'], 10)
        self.assertEqual(workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']['cmsRun1']['errorsBySite']['average']['99999'], 10)
        self.assertEqual(workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']['cmsRun1']['errorsBySite']['average']['8020'], 10)
        self.assertEqual(workloadSummary['histograms']['workflowLevel']['failuresBySite']['stdDev']['Failed Jobs'], 0)
        self.assertEqual(workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']['cmsRun1']['errorsBySite']['stdDev']['99999'], 0)
        self.assertEqual(workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']['cmsRun1']['errorsBySite']['stdDev']['8020'], 0)
        return
Esempio n. 10
0
class ChangeState(WMObject, WMConnectionBase):
    """
    Propagate the state of a job through the JSM.
    """

    def __init__(self, config, couchDbName=None):
        WMObject.__init__(self, config)
        WMConnectionBase.__init__(self, "WMCore.WMBS")

        if couchDbName == None:
            self.dbname = getattr(self.config.JobStateMachine, "couchDBName")
        else:
            self.dbname = couchDbName

        try:
            self.couchdb = CouchServer(self.config.JobStateMachine.couchurl)
            self.jobsdatabase = self.couchdb.connectDatabase("%s/jobs" % self.dbname)
            self.fwjrdatabase = self.couchdb.connectDatabase("%s/fwjrs" % self.dbname)
        except Exception, ex:
            logging.error("Error connecting to couch: %s" % str(ex))
            self.jobsdatabase = None
            self.fwjrdatabase = None

        try:
            self.dashboardReporter = DashboardReporter(config)
        except Exception, ex:
            logging.error(
                "Error setting up the \
                          dashboard reporter: %s"
                % str(ex)
            )
Esempio n. 11
0
class ChangeState(WMObject, WMConnectionBase):
    """
    Propagate the state of a job through the JSM.
    """
    def __init__(self, config, couchDbName = None):
        WMObject.__init__(self, config)
        WMConnectionBase.__init__(self, "WMCore.WMBS")

        if couchDbName == None:
            self.dbname = getattr(self.config.JobStateMachine, "couchDBName")
        else:
            self.dbname = couchDbName

        try:
            self.couchdb = CouchServer(self.config.JobStateMachine.couchurl)
            self.jobsdatabase = self.couchdb.connectDatabase("%s/jobs" % self.dbname)
            self.fwjrdatabase = self.couchdb.connectDatabase("%s/fwjrs" % self.dbname)
        except Exception, ex:
            logging.error("Error connecting to couch: %s" % str(ex))
            self.jobsdatabase = None
            self.fwjrdatabase = None            

        self.getCouchDAO = self.daofactory("Jobs.GetCouchID")
        self.setCouchDAO = self.daofactory("Jobs.SetCouchID")
        self.incrementRetryDAO = self.daofactory("Jobs.IncrementRetry")
        self.workflowTaskDAO = self.daofactory("Jobs.GetWorkflowTask")

        self.maxUploadedInputFiles = getattr(self.config.JobStateMachine, 'maxFWJRInputFiles', 1000)
        return
Esempio n. 12
0
class WorkQueueBackend(object):
    """
    Represents persistent storage for WorkQueue
    """
    def __init__(self,
                 db_url,
                 db_name='workqueue',
                 inbox_name=None,
                 parentQueue=None,
                 queueUrl=None,
                 logger=None):
        if logger:
            self.logger = logger
        else:
            import logging
            self.logger = logging

        if inbox_name == None:
            inbox_name = "%s_inbox" % db_name

        self.server = CouchServer(db_url)
        self.parentCouchUrlWithAuth = parentQueue
        if parentQueue:
            self.parentCouchUrl = sanitizeURL(parentQueue)['url']
        else:
            self.parentCouchUrl = None
        self.db = self.server.connectDatabase(db_name,
                                              create=False,
                                              size=10000)
        self.hostWithAuth = db_url
        self.inbox = self.server.connectDatabase(inbox_name,
                                                 create=False,
                                                 size=10000)
        self.queueUrl = sanitizeURL(queueUrl
                                    or (db_url + '/' + db_name))['url']

    def forceQueueSync(self):
        """Force a blocking replication
            - for use mainly in tests"""
        self.pullFromParent(continuous=False)
        self.sendToParent(continuous=False)

    def pullFromParent(self, continuous=True, cancel=False):
        """Replicate from parent couch - blocking"""
        try:
            if self.parentCouchUrl and self.queueUrl:
                self.server.replicate(source=self.parentCouchUrl,
                                      destination="%s/%s" %
                                      (self.hostWithAuth, self.inbox.name),
                                      filter='WorkQueue/queueFilter',
                                      query_params={
                                          'childUrl': self.queueUrl,
                                          'parentUrl': self.parentCouchUrl
                                      },
                                      continuous=continuous,
                                      cancel=cancel,
                                      useReplicator=True)
        except Exception, ex:
            self.logger.warning('Replication from %s failed: %s' %
                                (self.parentCouchUrl, str(ex)))
Esempio n. 13
0
    def testB_testErrors(self):
        """
        _testErrors_

        Test with a failed FWJR
        """

        myThread = threading.currentThread()

        config = self.getConfig()
        workloadPath = os.path.join(self.testDir, 'specDir', 'spec.pkl')
        workload     = self.createWorkload(workloadName = workloadPath)
        testJobGroup = self.createTestJobGroup(config = config,
                                               name = workload.name(),
                                               specLocation = workloadPath,
                                               error = True)

        cachePath = os.path.join(config.JobCreator.jobCacheDir,
                                 "TestWorkload", "ReReco")
        os.makedirs(cachePath)
        self.assertTrue(os.path.exists(cachePath))

        couchdb      = CouchServer(config.JobStateMachine.couchurl)
        jobdb        = couchdb.connectDatabase("%s/jobs" % self.databaseName)
        fwjrdb       = couchdb.connectDatabase("%s/fwjrs" % self.databaseName)
        jobdb.loadView("JobDump", "jobsByWorkflowName",
                        options = {"startkey": [workload.name()],
                                   "endkey": [workload.name(), {}]})['rows']
        fwjrdb.loadView("FWJRDump", "fwjrsByWorkflowName",
                        options = {"startkey": [workload.name()],
                                   "endkey": [workload.name(), {}]})['rows']

        testTaskArchiver = TaskArchiverPoller(config = config)
        testTaskArchiver.algorithm()

        dbname       = getattr(config.JobStateMachine, "couchDBName")
        workdatabase = couchdb.connectDatabase("%s/workloadsummary" % dbname)

        workloadSummary = workdatabase.document(id = workload.name())

        self.assertEqual(workloadSummary['errors']['/TestWorkload/ReReco']['failureTime'], 500)
        self.assertTrue(workloadSummary['errors']['/TestWorkload/ReReco']['cmsRun1'].has_key('99999'))
        self.assertEquals(workloadSummary['errors']['/TestWorkload/ReReco']['cmsRun1']['99999']['runs'], {'10' : [12312]},
                          "Wrong lumi information in the summary for failed jobs")

        # Check the failures by site histograms
        self.assertEqual(workloadSummary['histograms']['workflowLevel']['failuresBySite']['data']['T1_IT_CNAF']['Failed Jobs'], 10)
        self.assertEqual(workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']['cmsRun1']['errorsBySite']['data']['T1_IT_CNAF']['99999'], 10)
        self.assertEqual(workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']['cmsRun1']['errorsBySite']['data']['T1_IT_CNAF']['8020'], 10)
        self.assertEqual(workloadSummary['histograms']['workflowLevel']['failuresBySite']['average']['Failed Jobs'], 10)
        self.assertEqual(workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']['cmsRun1']['errorsBySite']['average']['99999'], 10)
        self.assertEqual(workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']['cmsRun1']['errorsBySite']['average']['8020'], 10)
        self.assertEqual(workloadSummary['histograms']['workflowLevel']['failuresBySite']['stdDev']['Failed Jobs'], 0)
        self.assertEqual(workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']['cmsRun1']['errorsBySite']['stdDev']['99999'], 0)
        self.assertEqual(workloadSummary['histograms']['stepLevel']['/TestWorkload/ReReco']['cmsRun1']['errorsBySite']['stdDev']['8020'], 0)
        return
Esempio n. 14
0
    def __init__(self, config):
        BaseDaemon.__init__(self, config, 'Analytics')

        server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
        self.db = server.connectDatabase(self.config.files_database)
        self.logger.debug('Connected to local couchDB')
        config_server = CouchServer(dburl=self.config.config_couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
        self.config_db = config_server.connectDatabase(self.config.config_database)
        self.amq_auth_file = self.config.amq_auth_file
        monitoring_server = CouchServer(dburl=self.config.couch_user_monitoring_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
        self.monitoring_db = monitoring_server.connectDatabase(self.config.user_monitoring_db)
        self.logger.debug('Connected to user_monitoring_db in couchDB')
Esempio n. 15
0
class WMLoggingTest(unittest.TestCase):
    def setUp(self):
        # Make an instance of the server
        self.server = CouchServer(os.getenv("COUCHURL", 'http://*****:*****@localhost:5984'))
        testname = self.id().split('.')[-1]
        # Create a database, drop an existing one first
        self.dbname = 'cmscouch_unittest_%s' % testname.lower()

        if self.dbname in self.server.listDatabases():
            self.server.deleteDatabase(self.dbname)

        self.server.createDatabase(self.dbname)
        self.db = self.server.connectDatabase(self.dbname)

    def tearDown(self):
        # This used to test self._exc_info to only run on success. Broke in 2.7. Removed.
        self.server.deleteDatabase(self.dbname)

    def testLog(self):
        """
        Write ten log messages to the database at three different levels
        """
        my_logger = logging.getLogger('MyLogger')
        my_logger.setLevel(logging.DEBUG)
        handler = CouchHandler(self.server.url, self.dbname)
        formatter = logging.Formatter('%(message)s')
        handler.setFormatter(formatter)
        my_logger.addHandler(handler)

        for _ in range(10):
            my_logger.debug('This is probably all noise.')
            my_logger.info('Jackdaws love my big sphinx of quartz.')
            my_logger.error('HOLLY CRAP!')
        logs = self.db.allDocs()['rows']
        self.assertEqual(30, len(logs))
Esempio n. 16
0
 def __init__(self, config, logger):
     self.config = config.CRABAsyncTransfer
     server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
     self.db = server.connectDatabase(self.config.files_database)
     self.logger = logger
     self.size = 0
     self.result = []
Esempio n. 17
0
class WMLoggingTest(unittest.TestCase):
    def setUp(self):
        # Make an instance of the server
        self.server = CouchServer(os.getenv("COUCHURL", 'http://*****:*****@localhost:5984'))
        testname = self.id().split('.')[-1]
        # Create a database, drop an existing one first
        self.dbname = 'cmscouch_unittest_%s' % testname.lower()

        if self.dbname in self.server.listDatabases():
            self.server.deleteDatabase(self.dbname)

        self.server.createDatabase(self.dbname)
        self.db = self.server.connectDatabase(self.dbname)

    def tearDown(self):
        # This used to test self._exc_info to only run on success. Broke in 2.7. Removed.
        self.server.deleteDatabase(self.dbname)

    def testLog(self):
        """
        Write ten log messages to the database at three different levels
        """
        my_logger = logging.getLogger('MyLogger')
        my_logger.setLevel(logging.DEBUG)
        handler = CouchHandler(self.server.url, self.dbname)
        formatter = logging.Formatter('%(message)s')
        handler.setFormatter(formatter)
        my_logger.addHandler(handler)

        for _ in range(10):
            my_logger.debug('This is probably all noise.')
            my_logger.info('Jackdaws love my big sphinx of quartz.')
            my_logger.error('HOLLY CRAP!')
        logs = self.db.allDocs()['rows']
        self.assertEqual(30, len(logs))
Esempio n. 18
0
    def setUp(self):
        """
        _setUp_

        Setup couchdb and the test environment
        """
        super(ResubmitBlockTest, self).setUp()

        self.group = 'unknown'
        self.user = '******'

        # Set external test helpers
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setupCouch("resubmitblock_t", "ACDC", "GroupUser")
        EmulatorHelper.setEmulators(phedex=False,
                                    dbs=False,
                                    siteDB=True,
                                    requestMgr=False)

        # Define test environment
        self.couchUrl = os.environ["COUCHURL"]
        self.acdcDBName = 'resubmitblock_t'
        self.validLocations = ['T2_US_Nebraska', 'T1_US_FNAL', 'T1_UK_RAL']
        self.siteWhitelist = ['T2_XX_SiteA']
        self.workflowName = 'dballest_ReReco_workflow'
        couchServer = CouchServer(dburl=self.couchUrl)
        self.acdcDB = couchServer.connectDatabase(self.acdcDBName,
                                                  create=False)
        user = makeUser(self.group, '*****@*****.**', self.couchUrl,
                        self.acdcDBName)
        user.create()

        return
Esempio n. 19
0
    def __init__(self, config):
        BaseWorkerThread.__init__(self)
        self.config = config.FilesCleaner
        self.logger.debug('Configuration loaded')

        try:
            self.logger.setLevel(self.config.log_level)
        except:
            import logging
            self.logger = logging.getLogger()
            self.logger.setLevel(self.config.log_level)
        self.logger.debug('Configuration loaded')
        config_server = CouchServer(dburl=self.config.config_couch_instance)
        self.config_db = config_server.connectDatabase(self.config.config_database)
        self.logger.debug('Connected to files DB')
        self.phedex = PhEDEx(responseType='xml')
        self.log_dir = '%s/logs/%s/%s/%s' % ( self.config.componentDir, \
 str(datetime.datetime.now().month), str(datetime.datetime.now().year), "Ops")
        try:
            os.makedirs(self.log_dir)
        except OSError, e:
            if e.errno == errno.EEXIST:
                pass
            else:
                self.logger.error('Unknown error in mkdir' % e.errno)
                raise
Esempio n. 20
0
    def atestB_testErrors(self):
        """
        _testErrors_

        Test with a failed FWJR
        """

        myThread = threading.currentThread()

        config = self.getConfig()
        workloadPath = os.path.join(self.testDir, "specDir", "spec.pkl")
        workload = self.createWorkload(workloadName=workloadPath)
        testJobGroup = self.createTestJobGroup(
            config=config, name=workload.name(), specLocation=workloadPath, error=True
        )

        cachePath = os.path.join(config.JobCreator.jobCacheDir, "TestWorkload", "ReReco")
        os.makedirs(cachePath)
        self.assertTrue(os.path.exists(cachePath))

        testTaskArchiver = TaskArchiverPoller(config=config)
        testTaskArchiver.algorithm()

        dbname = getattr(config.JobStateMachine, "couchDBName")
        couchdb = CouchServer(config.JobStateMachine.couchurl)
        workdatabase = couchdb.connectDatabase(dbname)

        workloadSummary = workdatabase.document(id="TestWorkload")

        self.assertEqual(workloadSummary["/TestWorkload/ReReco"]["failureTime"], 500)
        self.assertTrue(workloadSummary["/TestWorkload/ReReco"]["cmsRun1"].has_key("99999"))
        return
Esempio n. 21
0
    def saveCouch(self, couchUrl, couchDBName, metadata=None):
        """ Save this spec in CouchDB.  Returns URL """
        from WMCore.Database.CMSCouch import CouchServer, CouchInternalServerError
        metadata = metadata or {}
        server = CouchServer(couchUrl)
        database = server.connectDatabase(couchDBName)
        name = self.name()
        uri = '/%s/%s' % (couchDBName, name)
        specuri = uri + '/spec'
        if not database.documentExists(name):
            self.setSpecUrl(couchUrl + specuri)
            doc = database.put(uri, data=metadata, contentType='application/json')
            # doc = database.commitOne(self.name(), metadata)
            rev = doc['rev']
        else:
            # doc = database.get(uri+'?revs=true')
            doc = database.document(name)
            rev = doc['_rev']

        # specuriwrev = specuri + '?rev=%s' % rev
        workloadString = pickle.dumps(self.data)
        # result = database.put(specuriwrev, workloadString, contentType='application/text')
        retval = database.addAttachment(name, rev, workloadString, 'spec')
        if retval.get('ok', False) is not True:
            msg = "Failed to save a spec attachment in CouchDB for %s" % name
            raise CouchInternalServerError(msg, data=None, result=retval)

        url = couchUrl + specuri
        return url
Esempio n. 22
0
    def setUp(self):
        """
        _setUp_

        Setup couchdb and the test environment
        """
        # Set external test helpers
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setupCouch("resubmitblock_t", "ACDC", "GroupUser")
        EmulatorHelper.setEmulators(siteDB = True)

        # Define test environment
        self.couchUrl = os.environ["COUCHURL"]
        self.acdcDBName = 'resubmitblock_t'
        self.validLocations = ['srm-cms.gridpp.rl.ac.uk', 'cmssrm.fnal.gov', 'srm.unl.edu']
        self.validLocationsCMSNames = ['T2_US_Nebraska', 'T1_US_FNAL', 'T1_UK_RAL']
        self.siteWhitelist = ['T2_XX_SiteA']
        self.workflowName = 'dballest_ReReco_workflow'
        couchServer = CouchServer(dburl = self.couchUrl)
        self.acdcDB = couchServer.connectDatabase(self.acdcDBName, create = False)
        user = makeUser('unknown', '*****@*****.**', self.couchUrl, self.acdcDBName)
        user.create()

        return
Esempio n. 23
0
    def setUp(self):
        """
        _setUp_

        Setup couchdb and the test environment
        """
        super(ResubmitBlockTest, self).setUp()

        self.group = 'unknown'
        self.user = '******'

        # Set external test helpers
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setupCouch("resubmitblock_t", "ACDC", "GroupUser")
        EmulatorHelper.setEmulators(phedex=False, dbs=False, siteDB=True, requestMgr=False)

        # Define test environment
        self.couchUrl = os.environ["COUCHURL"]
        self.acdcDBName = 'resubmitblock_t'
        self.validLocations = ['T2_US_Nebraska', 'T1_US_FNAL', 'T1_UK_RAL']
        self.siteWhitelist = ['T2_XX_SiteA']
        self.workflowName = 'dballest_ReReco_workflow'
        couchServer = CouchServer(dburl=self.couchUrl)
        self.acdcDB = couchServer.connectDatabase(self.acdcDBName, create=False)
        user = makeUser(self.group, '*****@*****.**', self.couchUrl, self.acdcDBName)
        user.create()

        return
Esempio n. 24
0
    def saveCouch(self, couchUrl, couchDBName, metadata=None):
        """ Save this spec in CouchDB.  Returns URL """
        from WMCore.Database.CMSCouch import CouchServer, CouchInternalServerError
        metadata = metadata or {}
        server = CouchServer(couchUrl)
        database = server.connectDatabase(couchDBName)
        name = self.name()
        uri = '/%s/%s' % (couchDBName, name)
        specuri = uri + '/spec'
        if not database.documentExists(name):
            self.setSpecUrl(couchUrl + specuri)
            doc = database.put(uri, data=metadata, contentType='application/json')
            # doc = database.commitOne(self.name(), metadata)
            rev = doc['rev']
        else:
            # doc = database.get(uri+'?revs=true')
            doc = database.document(name)
            rev = doc['_rev']

        # specuriwrev = specuri + '?rev=%s' % rev
        # FIXME: once both central services and WMAgent are in Py3, we can remove protocol=0
        workloadString = pickle.dumps(self.data, protocol=0)
        # result = database.put(specuriwrev, workloadString, contentType='application/text')
        retval = database.addAttachment(name, rev, workloadString, 'spec')
        if retval.get('ok', False) is not True:
            msg = "Failed to save a spec attachment in CouchDB for %s" % name
            raise CouchInternalServerError(msg, data=None, result=retval)

        url = couchUrl + specuri
        return url
Esempio n. 25
0
    def setUp(self):
        """
        _setUp_

        Initialize the database and couch.
        """
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setupCouch("promptreco_t", "ConfigCache")
        self.testInit.setSchema(customModules=["WMCore.WMBS"],
                                useDefault=False)
        couchServer = CouchServer(os.environ["COUCHURL"])
        self.configDatabase = couchServer.connectDatabase("promptreco_t")
        self.testDir = self.testInit.generateWorkDir()

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package="WMCore.WMBS",
                                     logger=myThread.logger,
                                     dbinterface=myThread.dbi)
        self.listTasksByWorkflow = self.daoFactory(classname="Workflow.LoadFromName")
        self.listFilesets = self.daoFactory(classname="Fileset.List")
        self.listSubsMapping = self.daoFactory(classname="Subscriptions.ListSubsAndFilesetsFromWorkflow")

        return
Esempio n. 26
0
    def setUp(self):
        """
        _setUp_

        Create two subscriptions: One that contains a single file and one that
        contains multiple files.
        """
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.couchUrl = os.environ["COUCHURL"]
        self.couchDBName = "acdc_event_based_t"
        self.testInit.setupCouch(self.couchDBName, "GroupUser", "ACDC")
        self.testInit.setSchema(customModules=["WMCore.WMBS"],
                                useDefault=False)
        couchSever = CouchServer(dburl=self.couchUrl)
        self.couchDB = couchSever.connectDatabase(self.couchDBName)
        self.populateWMBS()
        self.performanceParams = {
            'timePerEvent': 12,
            'memoryRequirement': 2300,
            'sizePerEvent': 400
        }

        return
Esempio n. 27
0
class DQMCouchAPI(WMObject, WMConnectionBase):
    """
    Update the harvesting status of a dataset in CouchDB
    """
    def __init__(self, config, couchDbName = None, couchurl = None):
        WMObject.__init__(self, config)
        WMConnectionBase.__init__(self, "WMCore.WMBS")
        self.designDoc = "HarvestingDatasets"

        if couchDbName == None:
            self.dbname = getattr(self.config.HarvestingScheduler, "couchDBName",
                                  "dqm_default")
        else:
            self.dbname = couchDbName

        if couchurl is not None:
            self.couchurl = couchurl
        elif getattr(self.config.HarvestingScheduler, "couchurl",
                                                            None) is not None:
            self.couchurl = self.config.HarvestingScheduler.couchurl
        else:
            self.couchurl = self.config.JobStateMachine.couchurl

        try:
            self.couchdb = CouchServer(self.couchurl)
            if self.dbname not in self.couchdb.listDatabases():
                self.createDatabase()

            self.database = self.couchdb.connectDatabase(self.dbname,
                                                         size=_LIMIT)
        except Exception, ex:
            logging.error("Error connecting to couch: %s" % str(ex))
            self.database = None

        return 
Esempio n. 28
0
    def setUp(self):
        """
        _setUp_

        Setup couchdb and the test environment
        """
        super(ResubmitBlockTest, self).setUp()

        self.group = 'unknown'
        self.user = '******'

        # Set external test helpers
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setupCouch("resubmitblock_t", "ACDC", "GroupUser")

        # Define test environment
        self.couchUrl = os.environ["COUCHURL"]
        self.acdcDBName = 'resubmitblock_t'
        self.validLocations = [
            'T2_US_Nebraska', 'T1_US_FNAL_Disk', 'T1_UK_RAL_Disk'
        ]
        self.siteWhitelist = ['T2_XX_SiteA']
        cric = CRIC()
        # Convert phedex node name to a valid processing site name
        self.PSNs = cric.PNNstoPSNs(self.validLocations)
        self.workflowName = 'dballest_ReReco_workflow'
        couchServer = CouchServer(dburl=self.couchUrl)
        self.acdcDB = couchServer.connectDatabase(self.acdcDBName,
                                                  create=False)
        user = makeUser(self.group, '*****@*****.**', self.couchUrl,
                        self.acdcDBName)
        user.create()

        return
Esempio n. 29
0
    def __init__(self, config):
        """
        Initialise class members
        """
        BaseDaemon.__init__(self, config, 'RetryManager')

        if self.config.isOracle:
            self.oracleDB = HTTPRequests(self.config.oracleDB,
                                         self.config.opsProxy,
                                         self.config.opsProxy)
        else:
            try:
                server = CouchServer(dburl=self.config.couch_instance,
                                     ckey=self.config.opsProxy,
                                     cert=self.config.opsProxy)
                self.db = server.connectDatabase(self.config.files_database)
            except Exception as e:
                self.logger.exception('A problem occured when connecting to couchDB: %s' % e)
                raise
            self.logger.debug('Connected to files DB')

            # Set up a factory for loading plugins
        self.factory = WMFactory(self.config.retryAlgoDir, namespace=self.config.retryAlgoDir)
        try:
            self.plugin = self.factory.loadObject(self.config.algoName, self.config,
                                                  getFromCache=False, listFlag=True)
        except Exception as ex:
            msg = "Error loading plugin %s on path %s\n" % (self.config.algoName,
                                                            self.config.retryAlgoDir)
            msg += str(ex)
            self.logger.error(msg)
            raise RetryManagerException(msg)
        self.cooloffTime = self.config.cooloffTime
Esempio n. 30
0
 def __init__(self, config, logger):
     self.config = config.CRABAsyncTransfer
     server = CouchServer(self.config.couch_instance)
     self.db = server.connectDatabase(self.config.files_database)
     self.logger = logger
     self.size = 0
     self.result = []
Esempio n. 31
0
    def __init__(self, config):
        BaseWorkerThread.__init__(self)
        self.config = config.FilesCleaner
        self.logger.debug('Configuration loaded')

        try:
            self.logger.setLevel(self.config.log_level)
        except:
            import logging
            self.logger = logging.getLogger()
            self.logger.setLevel(self.config.log_level)
        self.logger.debug('Configuration loaded')
        config_server = CouchServer(dburl=self.config.config_couch_instance)
        self.config_db = config_server.connectDatabase(
            self.config.config_database)
        self.logger.debug('Connected to files DB')
        self.phedex = PhEDEx(responseType='xml')
        self.log_dir = '%s/logs/%s/%s/%s' % ( self.config.componentDir, \
 str(datetime.datetime.now().month), str(datetime.datetime.now().year), "Ops")
        try:
            os.makedirs(self.log_dir)
        except OSError, e:
            if e.errno == errno.EEXIST:
                pass
            else:
                self.logger.error('Unknown error in mkdir' % e.errno)
                raise
Esempio n. 32
0
 def resubmitCouchPublication(self, asourl, asodb, proxy, taskname):
     """
     Resubmit failed publications by resetting the publication
     status in the CouchDB documents.
     """
     server = CouchServer(dburl=asourl, ckey=proxy, cert=proxy)
     try:
         database = server.connectDatabase(asodb)
     except Exception as ex:
         msg = "Error while trying to connect to CouchDB: %s" % (str(ex))
         raise Exception(msg)
     try:
         failedPublications = database.loadView('DBSPublisher', 'PublicationFailedByWorkflow', {'reduce': False, 'startkey': [taskname], 'endkey': [taskname, {}]})['rows']
     except Exception as ex:
         msg = "Error while trying to load view 'DBSPublisher.PublicationFailedByWorkflow' from CouchDB: %s" % (str(ex))
         raise Exception(msg)
     msg = "There are %d failed publications to resubmit: %s" % (len(failedPublications), failedPublications)
     self.logger.info(msg)
     for doc in failedPublications:
         docid = doc['id']
         if doc['key'][0] != taskname: # this should never happen...
             msg = "Skipping document %s as it seems to correspond to another task: %s" % (docid, doc['key'][0])
             self.logger.warning(msg)
             continue
         data = {'last_update': time.time(),
                 'retry': str(datetime.datetime.now()),
                 'publication_state': 'not_published',
                }
         try:
             database.updateDocument(docid, 'DBSPublisher', 'updateFile', data)
             self.logger.info("updating document %s ", docid)
         except Exception as ex:
             self.logger.error("Error updating document %s in CouchDB: %s", docid, str(ex))
     return
Esempio n. 33
0
    def __init__(self, config):

        BaseWorkerThread.__init__(self)

        self.config = config.AsyncTransfer

        # self.logger is set up by the BaseWorkerThread, we just set it's level
        try:
            self.logger.setLevel(self.config.log_level)
        except:
            import logging
            self.logger = logging.getLogger()
            self.logger.setLevel(self.config.log_level)

        self.logger.debug('Configuration loaded')

        # Set up a factory for loading plugins
        self.factory = WMFactory(self.config.pluginDir, namespace = self.config.pluginDir)

        # Asynch db
        server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
        self.db = server.connectDatabase(self.config.files_database)
        self.logger.debug('Connected to CouchDB')

        return
Esempio n. 34
0
    def testMultiCoreReport(self):
        """
        _testMultiCoreReport_

        Verify that multicore reports can be json encoded and uploaded to couch.
        """
        couchdb = CouchServer(os.environ["COUCHURL"])
        fwjrdatabase = couchdb.connectDatabase("report_t/fwjrs")

        self.mcPath = os.path.join(
            WMCore.WMBase.getTestBase(),
            "WMCore_t/FwkJobReport_t/MulticoreReport.pkl")
        myReport = Report()
        myReport.unpersist(self.mcPath)

        fwjrDocument = {
            "_id": "303-0",
            "jobid": 303,
            "retrycount": 0,
            "fwjr": myReport.__to_json__(None),
            "type": "fwjr"
        }

        fwjrdatabase.queue(fwjrDocument, timestamp=True)
        fwjrdatabase.commit()
        return
Esempio n. 35
0
 def resubmitCouchPublication(self, asourl, asodb, proxy, taskname):
     """
     Resubmit failed publications by resetting the publication
     status in the CouchDB documents.
     """
     server = CouchServer(dburl=asourl, ckey=proxy, cert=proxy)
     try:
         database = server.connectDatabase(asodb)
     except Exception as ex:
         msg = "Error while trying to connect to CouchDB: %s" % (str(ex))
         raise Exception(msg)
     try:
         failedPublications = database.loadView('DBSPublisher', 'PublicationFailedByWorkflow', {'reduce': False, 'startkey': [taskname], 'endkey': [taskname, {}]})['rows']
     except Exception as ex:
         msg = "Error while trying to load view 'DBSPublisher.PublicationFailedByWorkflow' from CouchDB: %s" % (str(ex))
         raise Exception(msg)
     msg = "There are %d failed publications to resubmit: %s" % (len(failedPublications), failedPublications)
     self.logger.info(msg)
     for doc in failedPublications:
         docid = doc['id']
         if doc['key'][0] != taskname: # this should never happen...
             msg = "Skipping document %s as it seems to correspond to another task: %s" % (docid, doc['key'][0])
             self.logger.warning(msg)
             continue
         data = {'last_update': time.time(),
                 'retry': str(datetime.datetime.now()),
                 'publication_state': 'not_published',
                }
         try:
             database.updateDocument(docid, 'DBSPublisher', 'updateFile', data)
             self.logger.info("updating document %s " % docid)
         except Exception as ex:
             msg = "Error updating document %s in CouchDB: %s" % (docid, str(ex))
             self.logger.error(msg)
     return
Esempio n. 36
0
    def setUp(self):
        """
        _setUp_

        Setup couchdb and the test environment
        """
        super(ResubmitBlockTest, self).setUp()

        self.group = 'unknown'
        self.user = '******'

        # Set external test helpers
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setupCouch("resubmitblock_t", "ACDC", "GroupUser")

        # Define test environment
        self.couchUrl = os.environ["COUCHURL"]
        self.acdcDBName = 'resubmitblock_t'
        self.validLocations = ['T2_US_Nebraska', 'T1_US_FNAL_Disk', 'T1_UK_RAL_Disk']
        self.siteWhitelist = ['T2_XX_SiteA']
        siteDB = SiteDB()
        #Convert phedex node name to a valid processing site name
        self.PSNs = siteDB.PNNstoPSNs(self.validLocations)
        self.workflowName = 'dballest_ReReco_workflow'
        couchServer = CouchServer(dburl=self.couchUrl)
        self.acdcDB = couchServer.connectDatabase(self.acdcDBName, create=False)
        user = makeUser(self.group, '*****@*****.**', self.couchUrl, self.acdcDBName)
        user.create()

        return
Esempio n. 37
0
    def setUp(self):
        """
        _setUp_

        Initialize the database and couch.
        """
        super(MonteCarloTest, self).setUp()

        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setupCouch(TEST_DB_NAME, "ConfigCache")
        self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False)
        self.testInit.generateWorkDir()

        couchServer = CouchServer(os.environ["COUCHURL"])
        self.configDatabase = couchServer.connectDatabase(TEST_DB_NAME)

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package="WMCore.WMBS",
                                     logger=myThread.logger,
                                     dbinterface=myThread.dbi)
        self.listTasksByWorkflow = self.daoFactory(classname="Workflow.LoadFromName")
        self.listFilesets = self.daoFactory(classname="Fileset.List")
        self.listSubsMapping = self.daoFactory(classname="Subscriptions.ListSubsAndFilesetsFromWorkflow")

        return
Esempio n. 38
0
    def setUp(self):
        """
        _setUp_

        Initialize the database and couch.
        """
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setupCouch("promptreco_t", "ConfigCache")
        self.testInit.setSchema(customModules=["WMCore.WMBS"],
                                useDefault=False)
        couchServer = CouchServer(os.environ["COUCHURL"])
        self.configDatabase = couchServer.connectDatabase("promptreco_t")
        self.testDir = self.testInit.generateWorkDir()

        myThread = threading.currentThread()
        self.daoFactory = DAOFactory(package="WMCore.WMBS",
                                     logger=myThread.logger,
                                     dbinterface=myThread.dbi)
        self.listTasksByWorkflow = self.daoFactory(classname="Workflow.LoadFromName")
        self.listFilesets = self.daoFactory(classname="Fileset.List")
        self.listSubsMapping = self.daoFactory(classname="Subscriptions.ListSubsAndFilesetsFromWorkflow")
        self.promptSkim = None
        if PY3:
            self.assertItemsEqual = self.assertCountEqual
        return
Esempio n. 39
0
class WorkQueueBackend(object):
    """
    Represents persistent storage for WorkQueue
    """
    def __init__(self, db_url, db_name = 'workqueue',
                 inbox_name = None, parentQueue = None,
                 queueUrl = None, logger = None):
        if logger:
            self.logger = logger
        else:
            import logging
            self.logger = logging
        
        if inbox_name == None:
            inbox_name = "%s_inbox" % db_name
            
        self.server = CouchServer(db_url)
        self.parentCouchUrlWithAuth = parentQueue
        if parentQueue:
            self.parentCouchUrl = sanitizeURL(parentQueue)['url']
        else:
            self.parentCouchUrl = None
        self.db = self.server.connectDatabase(db_name, create = False, size = 10000)
        self.hostWithAuth = db_url
        self.inbox = self.server.connectDatabase(inbox_name, create = False, size = 10000)
        self.queueUrl = sanitizeURL(queueUrl or (db_url + '/' + db_name))['url']

    def forceQueueSync(self):
        """Force a blocking replication
            - for use mainly in tests"""
        self.pullFromParent(continuous = False)
        self.sendToParent(continuous = False)

    def pullFromParent(self, continuous = True, cancel = False):
        """Replicate from parent couch - blocking"""
        try:
            if self.parentCouchUrl and self.queueUrl:
                self.server.replicate(source = self.parentCouchUrl,
                                      destination = "%s/%s" % (self.hostWithAuth, self.inbox.name),
                                      filter = 'WorkQueue/queueFilter',
                                      query_params = {'childUrl' : self.queueUrl, 'parentUrl' : self.parentCouchUrl},
                                      continuous = continuous,
                                      cancel = cancel,
                                      useReplicator = True)
        except Exception, ex:
            self.logger.warning('Replication from %s failed: %s' % (self.parentCouchUrl, str(ex)))
Esempio n. 40
0
    def __init__(self, config):
        BaseDaemon.__init__(self, config, 'Statistics')

        server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
        self.db = server.connectDatabase(self.config.files_database)

        config_server = CouchServer(dburl=self.config.config_couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
        self.config_db = config_server.connectDatabase(self.config.config_database)
        self.mon_db = server.connectDatabase(self.config.mon_database)
        self.logger.debug('Connected to CouchDB')

        statserver = CouchServer(self.config.couch_statinstance)
        self.statdb = statserver.connectDatabase(self.config.statitics_database)
        self.logger.debug('Connected to Stat CouchDB')

        self.iteration_docs = []
        self.exptime = None
Esempio n. 41
0
    def testJ_Resubmission(self):
        """
        _Resubmission_
 
        Test Resubmission
         
        """
        userName = '******'
        groupName = 'Li'
        teamName = 'Tang'
        schema = utils.getAndSetupSchema(self,
                                         userName=userName,
                                         groupName=groupName,
                                         teamName=teamName)
        schema['RequestType'] = "ReReco"
        configID = self.createConfig()
        schema["ConfigCacheID"] = configID
        schema["CouchDBName"] = self.couchDBName
        schema["CouchURL"] = os.environ.get("COUCHURL")
        schema["CouchWorkloadDBName"] = self.couchDBName

        result = self.jsonSender.put('request', schema)
        requestName = result[0]['RequestName']

        # user, group schema already set up
        schema = utils.getSchema(groupName=groupName, userName=userName)
        schema['RequestType'] = "Resubmission"
        schema["CouchWorkloadDBName"] = self.couchDBName

        try:
            raises = False
            result = self.jsonSender.put('request', schema)
        except HTTPException as ex:
            raises = True
            self.assertEqual(ex.status, 400)
            self.assertTrue(
                "Error in Workload Validation: Validation failed: InitialTaskPath is mendatory"
                in ex.result)
        self.assertTrue(raises)

        schema["InitialTaskPath"] = '/%s/DataProcessing' % requestName
        schema["ACDCServer"] = os.environ.get("COUCHURL")
        schema["ACDCDatabase"] = self.couchDBName
        schema["CollectionName"] = "SomeOtherName"

        # Here we just make sure that real result goes through
        result = self.jsonSender.put('request', schema)
        resubmitName = result[0]['RequestName']
        result = self.jsonSender.get('request/%s' % resubmitName)

        couchServer = CouchServer(self.testInit.couchUrl)
        reqmgrCouch = couchServer.connectDatabase(self.couchDBName)
        result = reqmgrCouch.loadView('ReqMgr', 'childresubmissionrequests',
                                      {}, [requestName])['rows']
        self.assertEqual(len(result), 1)
        self.assertEqual(result[0]['key'], requestName)
        self.assertEqual(result[0]['id'], resubmitName)
Esempio n. 42
0
    def setUp(self):
        """
        _setUp_

        Setup the database and logging connection.  Try to create all of the
        WMBS tables.  Also add some dummy locations.
        """
        super(WorkQueueTestCase, self).setUp()
        self.queueDB = 'workqueue_t'
        self.queueInboxDB = 'workqueue_t_inbox'
        self.globalQDB = 'workqueue_t_global'
        self.globalQInboxDB = 'workqueue_t_global_inbox'
        self.localQDB = 'workqueue_t_local'
        self.localQInboxDB = 'workqueue_t_local_inbox'
        self.localQDB2 = 'workqueue_t_local2'
        self.localQInboxDB2 = 'workqueue_t_local2_inbox'
        self.configCacheDB = 'workqueue_t_config_cache'
        self.logDBName = 'logdb_t'
        self.requestDBName = 'workqueue_t_reqmgr_workload_cache'
        
        self.setSchema()
        self.testInit = TestInit('WorkQueueTest')
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection(destroyAllDatabase=True)
        self.addCleanup(self.testInit.clearDatabase)
        self.addCleanup(logging.debug, 'Cleanup called clearDatabase()')
        self.testInit.setSchema(customModules = self.schema,
                                useDefault = False)
        self.testInit.setupCouch(self.queueDB, *self.couchApps)
        self.testInit.setupCouch(self.queueInboxDB, *self.couchApps)
        self.testInit.setupCouch(self.globalQDB, *self.couchApps)
        self.testInit.setupCouch(self.globalQInboxDB , *self.couchApps)
        self.testInit.setupCouch(self.localQDB, *self.couchApps)
        self.testInit.setupCouch(self.localQInboxDB, *self.couchApps)
        self.testInit.setupCouch(self.localQDB2, *self.couchApps)
        self.testInit.setupCouch(self.localQInboxDB2, *self.couchApps)
        self.testInit.setupCouch(self.configCacheDB, 'ConfigCache')
        self.testInit.setupCouch(self.logDBName, 'LogDB')
        self.testInit.setupCouch(self.requestDBName, 'ReqMgr')
        
        self.couchURL = os.environ.get("COUCHURL")
        couchServer = CouchServer(self.couchURL)
        self.configCacheDBInstance = couchServer.connectDatabase(self.configCacheDB)

        self.localCouchMonitor = CouchMonitor(self.couchURL)
        self.localCouchMonitor.deleteReplicatorDocs()
        self.addCleanup(self.localCouchMonitor.deleteReplicatorDocs)
        self.addCleanup(logging.debug, 'Cleanup called deleteReplicatorDocs()')
        self.addCleanup(self.testInit.tearDownCouch)
        self.addCleanup(logging.debug, 'Cleanup called tearDownCouch()')

        self.workDir = self.testInit.generateWorkDir()
        self.addCleanup(self.testInit.delWorkDir)
        self.addCleanup(logging.debug, 'Cleanup called delWorkDir()')

        return
Esempio n. 43
0
    def __init__(self, config):
        BaseWorkerThread.__init__(self)
        self.config = config.Analytics

        try:
            self.logger.setLevel(self.config.log_level)
        except:
            import logging
            self.logger = logging.getLogger()
            self.logger.setLevel(self.config.log_level)
        self.logger.debug('Configuration loaded')
        server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
        self.db = server.connectDatabase(self.config.files_database)
        self.logger.debug('Connected to local couchDB')
        self.config_db = server.connectDatabase(self.config.config_database)
        self.amq_auth_file = self.config.amq_auth_file
        monitoring_server = CouchServer(dburl=self.config.couch_user_monitoring_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
        self.monitoring_db = monitoring_server.connectDatabase(self.config.user_monitoring_db)
        self.logger.debug('Connected to user_monitoring_db in couchDB')
Esempio n. 44
0
    def testE_multicore(self):
        """
        _multicore_

        Create a workload summary based on the multicore job report
        """

        myThread = threading.currentThread()

        config = self.getConfig()
        workloadPath = os.path.join(self.testDir, 'specDir', 'spec.pkl')
        workload     = self.createWorkload(workloadName = workloadPath)
        testJobGroup = self.createTestJobGroup(config = config,
                                               name = workload.name(),
                                               specLocation = workloadPath,
                                               error = False,
                                               multicore = True)

        cachePath = os.path.join(config.JobCreator.jobCacheDir,
                                 "TestWorkload", "ReReco")
        os.makedirs(cachePath)
        self.assertTrue(os.path.exists(cachePath))

        workflowName = "TestWorkload"
        dbname       = config.TaskArchiver.workloadSummaryCouchDBName
        couchdb      = CouchServer(config.JobStateMachine.couchurl)
        workdatabase = couchdb.connectDatabase(dbname)
        jobdb        = couchdb.connectDatabase("%s/jobs" % self.databaseName)
        fwjrdb       = couchdb.connectDatabase("%s/fwjrs" % self.databaseName)

        testTaskArchiver = TaskArchiverPoller(config = config)
        testTaskArchiver.algorithm()

        workloadSummary = workdatabase.document(id = "TestWorkload")

        self.assertEqual(workloadSummary['performance']['/TestWorkload/ReReco']['cmsRun1']['minMergeTime']['average'],
                         5.7624950408900002)
        self.assertEqual(workloadSummary['performance']['/TestWorkload/ReReco']['cmsRun1']['numberOfMerges']['average'],
                         3.0)
        self.assertEqual(workloadSummary['performance']['/TestWorkload/ReReco']['cmsRun1']['averageProcessTime']['average'],
                         29.369966666700002)
        return
Esempio n. 45
0
    def __init__(self, config):
        BaseWorkerThread.__init__(self)
        self.config = config.AsyncTransfer

        try:
            self.logger.setLevel(self.config.log_level)
        except:
            import logging
            self.logger = logging.getLogger()
            self.logger.setLevel(self.config.log_level)

        self.logger.debug('Configuration loaded')

        server = CouchServer(self.config.couch_instance)
        self.db = server.connectDatabase(self.config.files_database)
        self.logger.debug('Connected to CouchDB')

        statserver = CouchServer(self.config.couch_statinstance)
        self.statdb = statserver.connectDatabase(self.config.statitics_database)
        self.logger.debug('Connected to Stat CouchDB')
Esempio n. 46
0
    def __init__(self, config):
        BaseWorkerThread.__init__(self)
        self.config = config.Analytics

        try:
            self.logger.setLevel(self.config.log_level)
        except:
            import logging
            self.logger = logging.getLogger()
            self.logger.setLevel(self.config.log_level)
        self.logger.debug('Configuration loaded')
        server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
        self.db = server.connectDatabase(self.config.files_database)
        self.logger.debug('Connected to local couchDB')
        config_server = CouchServer(dburl=self.config.config_couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
        self.config_db = config_server.connectDatabase(self.config.config_database)
        self.amq_auth_file = self.config.amq_auth_file
        monitoring_server = CouchServer(dburl=self.config.couch_user_monitoring_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
        self.monitoring_db = monitoring_server.connectDatabase(self.config.user_monitoring_db)
        self.logger.debug('Connected to user_monitoring_db in couchDB')
    def __init__(self, config):
        BaseWorkerThread.__init__(self)
        self.config = config.AsyncTransfer

        try:
            self.logger.setLevel(self.config.log_level)
        except:
            import logging
            self.logger = logging.getLogger()
            self.logger.setLevel(self.config.log_level)

        self.logger.debug('Configuration loaded')

        server = CouchServer(self.config.couch_instance)
        self.db = server.connectDatabase(self.config.files_database)
        self.logger.debug('Connected to local couchDB')

        monitoring_server = CouchServer(self.config.couch_user_monitoring_instance)
        self.monitoring_db = monitoring_server.connectDatabase(self.config.user_monitoring_db)
        self.logger.debug('Connected to user_monitoring_db in couchDB')
Esempio n. 48
0
    def testJ_Resubmission(self):
        """
        _Resubmission_
 
        Test Resubmission
         
        """
        userName     = '******'
        groupName    = 'Li'
        teamName     = 'Tang'
        schema       = utils.getAndSetupSchema(self,
                                               userName = userName,
                                               groupName = groupName,
                                               teamName = teamName)
        schema['RequestType'] = "ReReco"
        configID = self.createConfig()
        schema["ConfigCacheID"] = configID
        schema["CouchDBName"] = self.couchDBName
        schema["CouchURL"]    = os.environ.get("COUCHURL")
        schema["CouchWorkloadDBName"] = self.couchDBName
 
        result = self.jsonSender.put('request', schema)
        requestName = result[0]['RequestName']
 
        # user, group schema already set up
        schema = utils.getSchema(groupName = groupName, userName = userName)
        schema['RequestType'] = "Resubmission"
        schema["CouchWorkloadDBName"] = self.couchDBName
        
        try:
            raises = False
            result = self.jsonSender.put('request', schema)
        except HTTPException as ex:
            raises = True
            self.assertEqual(ex.status, 400)
            self.assertTrue("Error in Workload Validation: Validation failed: InitialTaskPath is mendatory" in ex.result)
        self.assertTrue(raises)
 
        schema["InitialTaskPath"]     = '/%s/DataProcessing' % requestName
        schema["ACDCServer"]          = os.environ.get("COUCHURL")
        schema["ACDCDatabase"]        = self.couchDBName
        schema["CollectionName"]      = "SomeOtherName"
 
        # Here we just make sure that real result goes through
        result = self.jsonSender.put('request', schema)
        resubmitName = result[0]['RequestName']
        result = self.jsonSender.get('request/%s' % resubmitName)
 
        couchServer = CouchServer(self.testInit.couchUrl)
        reqmgrCouch = couchServer.connectDatabase(self.couchDBName)
        result = reqmgrCouch.loadView('ReqMgr', 'childresubmissionrequests', {}, [requestName])['rows']
        self.assertEqual(len(result), 1)
        self.assertEqual(result[0]['key'], requestName)
        self.assertEqual(result[0]['id'], resubmitName)
Esempio n. 49
0
    def testE_multicore(self):
        """
        _multicore_

        Create a workload summary based on the multicore job report
        """

        myThread = threading.currentThread()

        config = self.getConfig()
        workloadPath = os.path.join(self.testDir, "specDir", "spec.pkl")
        workload = self.createWorkload(workloadName=workloadPath)
        testJobGroup = self.createTestJobGroup(
            config=config, name=workload.name(), specLocation=workloadPath, error=False, multicore=True
        )

        cachePath = os.path.join(config.JobCreator.jobCacheDir, "TestWorkload", "ReReco")
        os.makedirs(cachePath)
        self.assertTrue(os.path.exists(cachePath))

        dbname = config.TaskArchiver.workloadSummaryCouchDBName
        couchdb = CouchServer(config.JobStateMachine.couchurl)
        workdatabase = couchdb.connectDatabase(dbname)

        testTaskArchiver = TaskArchiverPoller(config=config)
        testTaskArchiver.algorithm()

        result = myThread.dbi.processData("SELECT * FROM wmbs_job")[0].fetchall()
        self.assertEqual(len(result), 0, "No job should have survived")
        result = myThread.dbi.processData("SELECT * FROM wmbs_subscription")[0].fetchall()
        self.assertEqual(len(result), 0)
        result = myThread.dbi.processData("SELECT * FROM wmbs_jobgroup")[0].fetchall()
        self.assertEqual(len(result), 0)
        result = myThread.dbi.processData("SELECT * FROM wmbs_file_details")[0].fetchall()
        self.assertEqual(len(result), 0)

        workloadSummary = workdatabase.document(id="TestWorkload")

        self.assertAlmostEquals(
            workloadSummary["performance"]["/TestWorkload/ReReco"]["cmsRun1"]["minMergeTime"]["average"],
            5.7624950408900002,
            places=2,
        )
        self.assertAlmostEquals(
            workloadSummary["performance"]["/TestWorkload/ReReco"]["cmsRun1"]["numberOfMerges"]["average"],
            3.0,
            places=2,
        )
        self.assertAlmostEquals(
            workloadSummary["performance"]["/TestWorkload/ReReco"]["cmsRun1"]["averageProcessTime"]["average"],
            29.369966666700002,
            places=2,
        )
        return
Esempio n. 50
0
    def setUp(self):
        """
        Initialize the database and couch.

        """
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setupCouch("relvalmc_t", "ConfigCache")
        self.testInit.setSchema(customModules = ["WMCore.WMBS"],
                                useDefault = False)
        couchServer = CouchServer(os.environ["COUCHURL"])
        self.configDatabase = couchServer.connectDatabase("relvalmc_t")
Esempio n. 51
0
    def setUp(self):
        """
        Initialize the database and couch.

        """
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setupCouch("pileupfetcher_t", "ConfigCache")
        self.testInit.setSchema(customModules=["WMCore.WMBS"],
                                useDefault=False)
        couchServer = CouchServer(os.environ["COUCHURL"])
        self.configDatabase = couchServer.connectDatabase("pileupfetcher_t")
        self.testDir = self.testInit.generateWorkDir()
        EmulatorHelper.setEmulators(dbs=True)
    def __init__(self, config):
        """
        Initialise class members
        """
        BaseDaemon.__init__(self, config, 'RetryManager')

        try:
            server = CouchServer(dburl=self.config.couch_instance,
                                 ckey=self.config.opsProxy,
                                 cert=self.config.opsProxy)
            self.db = server.connectDatabase(self.config.files_database)
        except Exception, e:
            self.logger.exception(
                'A problem occured when connecting to couchDB: %s' % e)
            raise
Esempio n. 53
0
    def __init__(self, config):
        """
        Initialise class members
        """
        #Need a better way to test this without turning off this next line
        BaseDaemon.__init__(self, config, 'DBSPublisher')

        server = CouchServer(dburl=self.config.couch_instance,
                             ckey=self.config.opsProxy,
                             cert=self.config.opsProxy)
        self.db = server.connectDatabase(self.config.files_database)
        self.logger.debug('Connected to CouchDB')
        # Set up a factory for loading plugins
        self.factory = WMFactory(self.config.schedAlgoDir,
                                 namespace=self.config.schedAlgoDir)
        self.pool = Pool(processes=self.config.publication_pool_size)
Esempio n. 54
0
class Registration():
    def __init__(self, cfg_dict={}, reg_info={}):
        """
        Initialise the regsvc for this component,
        """
        try:
            config_dict = {
                'server': 'https://cmsweb.cern.ch/',
                'database': 'registration',
                'cacheduration': 1,
            }

            config_dict.update(cfg_dict)

            self.server = CouchServer(config_dict['server'])
            self.db = self.server.connectDatabase(config_dict['database'])

            if 'location' not in reg_info.keys():
                raise KeyError('Registration needs a location in its reg_info')
            self.location_hash = str(reg_info['location'].__hash__())
            reg_info['_id'] = self.location_hash
            reg_info['#config_hash'] = hash(str(reg_info))
            push_cfg = True
            if self.db.documentExists(self.location_hash):
                # If the doc exists, check that the configuration hasn't changed
                doc = self.db.document(self.location_hash)
                push_cfg = doc['#config_hash'] != reg_info['#config_hash']
                reg_info['_rev'] = doc['_rev']
            if push_cfg:
                self.db.commitOne(reg_info)
        except:
            # Don't want to raise anything here
            # TODO: but should probably log...
            pass
        self.report()

    def report(self):
        """
        'Ping' the RegSvc with a doc containing the service doc's ID and a
        timestamp, this can be used to provide uptime information.
        """
        try:
            self.db.commitOne({'service': self.location_hash}, timestamp=True)
        except:
            # Don't want to raise anything here
            # TODO: but should probably log...
            pass
Esempio n. 55
0
    def setUp(self):
        """
        _setUp_

        Initialize the database.
        """
        self.testInit = TestInitCouchApp(__file__)
        self.testInit.setLogging()
        self.testInit.setDatabaseConnection()
        self.testInit.setupCouch("mclhe_t", "ConfigCache")
        self.testInit.setSchema(customModules=["WMCore.WMBS"],
                                useDefault=False)

        couchServer = CouchServer(os.environ["COUCHURL"])
        self.configDatabase = couchServer.connectDatabase("mclhe_t")
        self.testDir = self.testInit.generateWorkDir()
        return