def __init__(self, config): """ Initialise class members """ #Need a better way to test this without turning off this next line BaseWorkerThread.__init__(self) #logging.basicConfig(format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s',datefmt = '%m-%d %H:%M') #self.logger = logging.getLogger() # self.logger is set up by the BaseWorkerThread, we just set it's level self.config = config.AsyncTransfer try: self.logger.setLevel(self.config.log_level) except: import logging self.logger = logging.getLogger() self.logger.setLevel(self.config.log_level) self.logger.debug('Configuration loaded') server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy) self.db = server.connectDatabase(self.config.files_database) config_server = CouchServer(dburl=self.config.config_couch_instance) self.config_db = config_server.connectDatabase( self.config.config_database) self.logger.debug('Connected to CouchDB') self.pool = Pool(processes=self.config.pool_size) try: self.phedex = PhEDEx(responseType='xml') except Exception, e: self.logger.exception('PhEDEx exception: %s' % e)
def __init__(self, config): BaseWorkerThread.__init__(self) self.config = config.Statistics try: self.logger.setLevel(self.config.log_level) except: import logging self.logger = logging.getLogger() self.logger.setLevel(self.config.log_level) self.logger.debug('Configuration loaded') server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy) self.db = server.connectDatabase(self.config.files_database) config_server = CouchServer(dburl=self.config.config_couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy) self.config_db = config_server.connectDatabase( self.config.config_database) self.mon_db = server.connectDatabase(self.config.mon_database) self.logger.debug('Connected to CouchDB') statserver = CouchServer(self.config.couch_statinstance) self.statdb = statserver.connectDatabase( self.config.statitics_database) self.logger.debug('Connected to Stat CouchDB') self.iteration_docs = [] self.exptime = None
def __init__(self, couchURL, statSummaryDB, summaryLevel): # set the connection for local couchDB call self.couchURL = couchURL self.couchURLBase, self.dbName = splitCouchServiceURL(couchURL) self.jobCouchDB = CouchServer(self.couchURLBase).connectDatabase(self.dbName + "/jobs", False) self.fwjrsCouchDB = CouchServer(self.couchURLBase).connectDatabase(self.dbName + "/fwjrs", False) #TODO: remove the hard coded name (wma_summarydb) self.summaryStatsDB = CouchServer(self.couchURLBase).connectDatabase(statSummaryDB, False) self.summaryLevel = summaryLevel
def __init__(self, couchURL, summaryLevel): # set the connection for local couchDB call self.couchURL = couchURL self.couchURLBase, self.dbName = splitCouchServiceURL(couchURL) self.jobCouchDB = CouchServer(self.couchURLBase).connectDatabase( self.dbName + "/jobs", False) self.fwjrsCouchDB = CouchServer(self.couchURLBase).connectDatabase( self.dbName + "/fwjrs", False) self.summaryLevel = summaryLevel
def __init__(self, couchURL, statSummaryDB, summaryLevel): # set the connection for local couchDB call self.couchURL = couchURL self.couchURLBase, self.dbName = splitCouchServiceURL(couchURL) self.jobCouchDB = CouchServer(self.couchURLBase).connectDatabase(self.dbName + "/jobs", False) fwjrDBname = "%s/fwjrs" % self.dbName self.fwjrAPI = FWJRDBAPI(self.couchURLBase, fwjrDBname) self.summaryStatsDB = CouchServer(self.couchURLBase).connectDatabase(statSummaryDB, False) self.summaryLevel = summaryLevel
def __init__(self, couchURL, dbName = None): couchURL = sanitizeURL(couchURL)['url'] # set the connection for local couchDB call if dbName: self.couchURL = couchURL self.dbName = dbName else: self.couchURL, self.dbName = splitCouchServiceURL(couchURL) self.couchServer = CouchServer(self.couchURL) self.couchDB = CouchServer(self.couchURL).connectDatabase(self.dbName, False)
def __init__(self, config): BaseDaemon.__init__(self, config, 'Analytics') server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy) self.db = server.connectDatabase(self.config.files_database) self.logger.debug('Connected to local couchDB') config_server = CouchServer(dburl=self.config.config_couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy) self.config_db = config_server.connectDatabase(self.config.config_database) self.amq_auth_file = self.config.amq_auth_file monitoring_server = CouchServer(dburl=self.config.couch_user_monitoring_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy) self.monitoring_db = monitoring_server.connectDatabase(self.config.user_monitoring_db) self.logger.debug('Connected to user_monitoring_db in couchDB')
def setUp(self): myThread = threading.currentThread() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) self.databaseName = "couchapp_t_0" self.testInit.setupCouch(self.databaseName, "WorkloadSummary") self.testInit.setupCouch("%s/jobs" % self.databaseName, "JobDump") self.testInit.setupCouch("%s/fwjrs" % self.databaseName, "FWJRDump") # Setup config for couch connections config = self.testInit.getConfiguration() config.section_("JobStateMachine") config.JobStateMachine.couchurl = os.getenv("COUCHURL", "cmssrv52.fnal.gov:5984") config.JobStateMachine.couchDBName = self.databaseName # Create couch server and connect to databases self.couchdb = CouchServer(config.JobStateMachine.couchurl) self.jobsdatabase = self.couchdb.connectDatabase( "%s/jobs" % config.JobStateMachine.couchDBName) self.fwjrdatabase = self.couchdb.connectDatabase( "%s/fwjrs" % config.JobStateMachine.couchDBName) # Create changeState self.changeState = ChangeState(config) self.config = config # Create testDir self.testDir = self.testInit.generateWorkDir() return
def setUp(self): """ _setUp_ Initialize the database and couch. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("promptreco_t", "ConfigCache") self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) couchServer = CouchServer(os.environ["COUCHURL"]) self.configDatabase = couchServer.connectDatabase("promptreco_t") self.testDir = self.testInit.generateWorkDir() myThread = threading.currentThread() self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.listTasksByWorkflow = self.daoFactory(classname="Workflow.LoadFromName") self.listFilesets = self.daoFactory(classname="Fileset.List") self.listSubsMapping = self.daoFactory(classname="Subscriptions.ListSubsAndFilesetsFromWorkflow") self.promptSkim = None if PY3: self.assertItemsEqual = self.assertCountEqual return
def __init__(self, cfg_dict={}, reg_info={}): """ Initialise the regsvc for this component, """ try: config_dict = { 'server': 'https://cmsweb.cern.ch/', 'database': 'registration', 'cacheduration': 1, } config_dict.update(cfg_dict) self.server = CouchServer(config_dict['server']) self.db = self.server.connectDatabase(config_dict['database']) if 'location' not in reg_info.keys(): raise KeyError('Registration needs a location in its reg_info') self.location_hash = str(reg_info['location'].__hash__()) reg_info['_id'] = self.location_hash reg_info['#config_hash'] = hash(str(reg_info)) push_cfg = True if self.db.documentExists(self.location_hash): # If the doc exists, check that the configuration hasn't changed doc = self.db.document(self.location_hash) push_cfg = doc['#config_hash'] != reg_info['#config_hash'] reg_info['_rev'] = doc['_rev'] if push_cfg: self.db.commitOne(reg_info) except: # Don't want to raise anything here # TODO: but should probably log... pass self.report()
def __init__(self, config): # configuration values: # 'uri' attribute (URL of the REST server and resource name) # in case of CouchDB, the resource name is the database name # http://servername:port/databaseName self.config = config logging.info("Instantiating ...") # the class currently relies only on 1 REST server possibility - the # CouchDB server. as explained above, .database will be replaced by # .connection if both a generic REST server as well as CouchDB are to # be talked to split = self.config.uri.rfind('/') dbName = self.config.uri[split + 1:] # get last item of URI - database name url = self.config.uri[:split] # as opposed to CouchSink, here it's assumed the resource (the database name) # does exist, fail here otherwise # this check / rest of the constructed may be revised for # general REST server server = CouchServer(url) databases = server.listDatabases() # there needs to be this database created upfront and also # couchapp associated with it installed, if it's there, fail if dbName not in databases: raise Exception("REST URI: %s (DB name: %s) does not exist." % (self.config.uri, dbName)) self._database = Database(dbName, url) logging.info("Initialized.")
def saveCouch(self, couchUrl, couchDBName, metadata=None): """ Save this spec in CouchDB. Returns URL """ from WMCore.Database.CMSCouch import CouchServer, CouchInternalServerError metadata = metadata or {} server = CouchServer(couchUrl) database = server.connectDatabase(couchDBName) name = self.name() uri = '/%s/%s' % (couchDBName, name) specuri = uri + '/spec' if not database.documentExists(name): self.setSpecUrl(couchUrl + specuri) doc = database.put(uri, data=metadata, contentType='application/json') # doc = database.commitOne(self.name(), metadata) rev = doc['rev'] else: # doc = database.get(uri+'?revs=true') doc = database.document(name) rev = doc['_rev'] # specuriwrev = specuri + '?rev=%s' % rev # FIXME: once both central services and WMAgent are in Py3, we can remove protocol=0 workloadString = pickle.dumps(self.data, protocol=0) # result = database.put(specuriwrev, workloadString, contentType='application/text') retval = database.addAttachment(name, rev, workloadString, 'spec') if retval.get('ok', False) is not True: msg = "Failed to save a spec attachment in CouchDB for %s" % name raise CouchInternalServerError(msg, data=None, result=retval) url = couchUrl + specuri return url
def setUp(self): """ _setUp_ Setup couchdb and the test environment """ super(ResubmitBlockTest, self).setUp() self.group = 'unknown' self.user = '******' # Set external test helpers self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setupCouch("resubmitblock_t", "ACDC", "GroupUser") # Define test environment self.couchUrl = os.environ["COUCHURL"] self.acdcDBName = 'resubmitblock_t' self.validLocations = [ 'T2_US_Nebraska', 'T1_US_FNAL_Disk', 'T1_UK_RAL_Disk' ] self.siteWhitelist = ['T2_XX_SiteA'] cric = CRIC() # Convert phedex node name to a valid processing site name self.PSNs = cric.PNNstoPSNs(self.validLocations) self.workflowName = 'dballest_ReReco_workflow' couchServer = CouchServer(dburl=self.couchUrl) self.acdcDB = couchServer.connectDatabase(self.acdcDBName, create=False) user = makeUser(self.group, '*****@*****.**', self.couchUrl, self.acdcDBName) user.create() return
def __init__(self, db_url, db_name='workqueue', inbox_name=None, parentQueue=None, queueUrl=None, logger=None): if logger: self.logger = logger else: import logging self.logger = logging if inbox_name is None: inbox_name = "%s_inbox" % db_name self.server = CouchServer(db_url) self.parentCouchUrlWithAuth = parentQueue if parentQueue: self.parentCouchUrl = sanitizeURL(parentQueue)['url'] else: self.parentCouchUrl = None self.db = self.server.connectDatabase(db_name, create=False, size=10000) self.hostWithAuth = db_url self.inbox = self.server.connectDatabase(inbox_name, create=False, size=10000) self.queueUrl = sanitizeURL(queueUrl or (db_url + '/' + db_name))['url'] self.eleKey = 'WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement'
def main(options): url, dbName = splitCouchServiceURL(options.dburl) db = CouchServer(url).connectDatabase(dbName) reqmgr_requests = generate_reqmgr_requests(options.requests) agent_requests = generate_agent_requests(options.requests, options.iterations) if options.add_couchapp: installCouchApp(url, dbName, "WMStats", options.couchapp_base) if options.add_reqmgr_data: for req in reqmgr_requests: db.queue(req) db.commit() print("Added %s reqmgr requests" % len(reqmgr_requests)) if options.add_agent_data: for req in agent_requests: db.queue(req) jobDocs = generate_jobsummary(req['workflow']) for job in jobDocs: db.queue(job) db.commit() print("Added %s agent requests" % len(agent_requests)) print("Added %s job Docs" % (len(agent_requests) * len(jobDocs)))
def setUp(self): """ _setUp_ """ self.transitions = Transitions() self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch("changestate_t/jobs", "JobDump") self.testInit.setupCouch("changestate_t/fwjrs", "FWJRDump") self.testInit.setupCouch("job_summary", "WMStats") self.testInit.setSchema(customModules = ["WMCore.WMBS"], useDefault = False) myThread = threading.currentThread() self.daoFactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) couchurl = os.getenv("COUCHURL") self.couchServer = CouchServer(dburl = couchurl) self.config = self.testInit.getConfiguration() self.taskName = "/TestWorkflow/ReReco1" self.specGen = WMSpecGenerator() self.specUrl = self.specGen.createProcessingSpec("TestWorkflow", returnType="file") return
def _commonInit(self, couchURL, couchapp): """ setting up comon variables for inherited class. inherited class should call this in their init function """ if isinstance(couchURL, Database): self.couchDB = couchURL self.couchURL = self.couchDB['host'] self.dbName = self.couchDB.name self.couchServer = CouchServer(self.couchURL) else: self.couchURL, self.dbName = splitCouchServiceURL(couchURL) self.couchServer = CouchServer(self.couchURL) self.couchDB = self.couchServer.connectDatabase(self.dbName, False) self.couchapp = couchapp self.defaultStale = {"stale": "update_after"}
def __init__(self, db_url, db_name='workqueue', inbox_name='workqueue_inbox', parentQueue=None, queueUrl=None, logger=None): if logger: self.logger = logger else: import logging self.logger = logging self.server = CouchServer(db_url) self.parentCouchUrlWithAuth = parentQueue if parentQueue: self.parentCouchUrl = sanitizeURL(parentQueue)['url'] else: self.parentCouchUrl = None self.db = self.server.connectDatabase(db_name, create=False, size=10000) self.hostWithAuth = db_url self.inbox = self.server.connectDatabase(inbox_name, create=False, size=10000) self.queueUrl = sanitizeURL(queueUrl or (db_url + '/' + db_name))['url']
def __init__(self, config): BaseWorkerThread.__init__(self) self.config = config.FilesCleaner self.logger.debug('Configuration loaded') try: self.logger.setLevel(self.config.log_level) except: import logging self.logger = logging.getLogger() self.logger.setLevel(self.config.log_level) self.logger.debug('Configuration loaded') config_server = CouchServer(dburl=self.config.config_couch_instance) self.config_db = config_server.connectDatabase( self.config.config_database) self.logger.debug('Connected to files DB') self.phedex = PhEDEx(responseType='xml') self.log_dir = '%s/logs/%s/%s/%s' % ( self.config.componentDir, \ str(datetime.datetime.now().month), str(datetime.datetime.now().year), "Ops") try: os.makedirs(self.log_dir) except OSError, e: if e.errno == errno.EEXIST: pass else: self.logger.error('Unknown error in mkdir' % e.errno) raise
def __init__(self, db_url, db_name, identifier, thread_name, **kwds): self.db_url = db_url self.server = CouchServer(db_url) self.db_name = db_name self.dbid = identifier self.thread_name = thread_name self.agent = kwds.get('agent', 0) create = kwds.get('create', False) size = kwds.get('size', 10000) self.db = self.server.connectDatabase(db_name, create=create, size=size) self.design = kwds.get('design', 'LogDB') # name of design document self.view = kwds.get('view', 'requests') # name of view to look-up requests self.tsview = kwds.get('tsview', 'tstamp') # name of tsview to look-up requests if create: uri = '/%s/_design/%s' % (db_name, self.design) data = design_doc() try: # insert design doc, if fails due to conflict continue # conflict may happen due to concurrent client connection who # created first this doc self.db.put(uri, data) except CouchConflictError: pass
def testTimestampAccounting(self): """ _testTimestampAccounting_ Check the correct functioning of the timestamp view in the ACDC couchapp and the function to remove old filesets. """ self.populateCouchDB() svc = CouchService(url=self.testInit.couchUrl, database=self.testInit.couchDbName) currentTime = time.time() database = CouchServer(self.testInit.couchUrl).connectDatabase( self.testInit.couchDbName) results = database.loadView("ACDC", "byTimestamp", {"endkey": currentTime}) self.assertEqual(len(results["rows"]), 4) results = database.loadView("ACDC", "byTimestamp", {"endkey": currentTime - 2}) self.assertEqual(len(results["rows"]), 3) results = database.loadView("ACDC", "byTimestamp", {"endkey": currentTime - 3}) self.assertEqual(len(results["rows"]), 2) results = database.loadView("ACDC", "byTimestamp", {"endkey": currentTime - 4}) self.assertEqual(len(results["rows"]), 1) results = database.loadView("ACDC", "byTimestamp", {"endkey": currentTime - 5}) self.assertEqual(len(results["rows"]), 0) svc.removeOldFilesets(0) results = database.loadView("ACDC", "byTimestamp", {"endkey": currentTime}) self.assertEqual(len(results["rows"]), 0) return
def __init__(self, config, couchDbName=None): WMObject.__init__(self, config) WMConnectionBase.__init__(self, "WMCore.WMBS") if couchDbName == None: self.dbname = getattr(self.config.JobStateMachine, "couchDBName") else: self.dbname = couchDbName try: self.couchdb = CouchServer(self.config.JobStateMachine.couchurl) self.jobsdatabase = self.couchdb.connectDatabase("%s/jobs" % self.dbname, size=250) self.fwjrdatabase = self.couchdb.connectDatabase("%s/fwjrs" % self.dbname, size=250) self.jsumdatabase = self.couchdb.connectDatabase(getattr( self.config.JobStateMachine, 'jobSummaryDBName'), size=250) except Exception, ex: logging.error("Error connecting to couch: %s" % str(ex)) self.jobsdatabase = None self.fwjrdatabase = None self.jsumdatabase = None
def testRemoveByCollectionName(self): """ _testRemoveByCollectionName_ Check the function to obliterate all the filesets of a collection """ self.populateCouchDB() svc = CouchService(url=self.testInit.couchUrl, database=self.testInit.couchDbName) database = CouchServer(self.testInit.couchUrl).connectDatabase( self.testInit.couchDbName) results = database.loadView("ACDC", "byCollectionName", keys=["Thunderstruck"]) self.assertTrue(len(results["rows"]) > 0) svc.removeFilesetsByCollectionName("Thunderstruck") results = database.loadView("ACDC", "byCollectionName", keys=["Thunderstruck"]) self.assertEqual(len(results["rows"]), 0) results = database.loadView("ACDC", "byCollectionName", keys=["Struckthunder"]) self.assertTrue(len(results["rows"]) > 0) svc.removeFilesetsByCollectionName("Struckthunder") results = database.loadView("ACDC", "byCollectionName", keys=["Struckthunder"]) self.assertEqual(len(results["rows"]), 0) return
def __init__(self, config, couchDbName=None): WMObject.__init__(self, config) WMConnectionBase.__init__(self, "WMCore.WMBS") if couchDbName is None: self.dbname = getattr(self.config.JobStateMachine, "couchDBName") else: self.dbname = couchDbName self.jobsdatabase = None self.fwjrdatabase = None self.jsumdatabase = None self.statsumdatabase = None self.couchdb = CouchServer(self.config.JobStateMachine.couchurl) self._connectDatabases() try: self.dashboardReporter = DashboardReporter(config) except Exception as ex: logging.error("Error setting up the dashboard reporter: %s", str(ex)) raise self.getCouchDAO = self.daofactory("Jobs.GetCouchID") self.setCouchDAO = self.daofactory("Jobs.SetCouchID") self.incrementRetryDAO = self.daofactory("Jobs.IncrementRetry") self.workflowTaskDAO = self.daofactory("Jobs.GetWorkflowTask") self.jobTypeDAO = self.daofactory("Jobs.GetType") self.updateLocationDAO = self.daofactory("Jobs.UpdateLocation") self.getWorkflowSpecDAO = self.daofactory("Workflow.GetSpecAndNameFromTask") self.maxUploadedInputFiles = getattr(self.config.JobStateMachine, 'maxFWJRInputFiles', 1000) self.workloadCache = {} return
def resubmitCouchPublication(self, asourl, asodb, proxy, taskname): """ Resubmit failed publications by resetting the publication status in the CouchDB documents. """ server = CouchServer(dburl=asourl, ckey=proxy, cert=proxy) try: database = server.connectDatabase(asodb) except Exception as ex: msg = "Error while trying to connect to CouchDB: %s" % (str(ex)) raise Exception(msg) try: failedPublications = database.loadView('DBSPublisher', 'PublicationFailedByWorkflow', {'reduce': False, 'startkey': [taskname], 'endkey': [taskname, {}]})['rows'] except Exception as ex: msg = "Error while trying to load view 'DBSPublisher.PublicationFailedByWorkflow' from CouchDB: %s" % (str(ex)) raise Exception(msg) msg = "There are %d failed publications to resubmit: %s" % (len(failedPublications), failedPublications) self.logger.info(msg) for doc in failedPublications: docid = doc['id'] if doc['key'][0] != taskname: # this should never happen... msg = "Skipping document %s as it seems to correspond to another task: %s" % (docid, doc['key'][0]) self.logger.warning(msg) continue data = {'last_update': time.time(), 'retry': str(datetime.datetime.now()), 'publication_state': 'not_published', } try: database.updateDocument(docid, 'DBSPublisher', 'updateFile', data) self.logger.info("updating document %s ", docid) except Exception as ex: self.logger.error("Error updating document %s in CouchDB: %s", docid, str(ex)) return
def setUp(self): self.couchURL = os.getenv("COUCHURL") self.server = CouchServer(self.couchURL) # Kill off any databases left over from previous runs for db in [ db for db in self.server.listDatabases() if db.startswith('rotdb_unittest_') ]: try: self.server.deleteDatabase(db) except: pass # Create a database, drop an existing one first testname = self.id().split('.')[-1].lower() self.dbname = 'rotdb_unittest_%s' % testname self.arcname = 'rotdb_unittest_%s_archive' % testname self.seedname = 'rotdb_unittest_%s_seedcfg' % testname # set a long value for times, tests do operations explicitly self.timing = { 'archive': timedelta(seconds=1), 'expire': timedelta(seconds=2) } self.db = RotatingDatabase(dbname=self.dbname, url=self.couchURL, archivename=self.arcname, timing=self.timing)
def setUp(self): """ _setUp_ Initialize the database and couch. """ super(MonteCarloTest, self).setUp() self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setupCouch(TEST_DB_NAME, "ConfigCache") self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) self.testInit.generateWorkDir() couchServer = CouchServer(os.environ["COUCHURL"]) self.configDatabase = couchServer.connectDatabase(TEST_DB_NAME) myThread = threading.currentThread() self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.listTasksByWorkflow = self.daoFactory(classname="Workflow.LoadFromName") self.listFilesets = self.daoFactory(classname="Fileset.List") self.listSubsMapping = self.daoFactory(classname="Subscriptions.ListSubsAndFilesetsFromWorkflow") return
def __init__(self, config): BaseWorkerThread.__init__(self) self.config = config.AsyncTransfer # self.logger is set up by the BaseWorkerThread, we just set it's level try: self.logger.setLevel(self.config.log_level) except: import logging self.logger = logging.getLogger() self.logger.setLevel(self.config.log_level) self.logger.debug('Configuration loaded') # Set up a factory for loading plugins self.factory = WMFactory(self.config.pluginDir, namespace = self.config.pluginDir) # Asynch db server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy) self.db = server.connectDatabase(self.config.files_database) self.logger.debug('Connected to CouchDB') return
def testMultiCoreReport(self): """ _testMultiCoreReport_ Verify that multicore reports can be json encoded and uploaded to couch. """ couchdb = CouchServer(os.environ["COUCHURL"]) fwjrdatabase = couchdb.connectDatabase("report_t/fwjrs") self.mcPath = os.path.join( WMCore.WMBase.getTestBase(), "WMCore_t/FwkJobReport_t/MulticoreReport.pkl") myReport = Report() myReport.unpersist(self.mcPath) fwjrDocument = { "_id": "303-0", "jobid": 303, "retrycount": 0, "fwjr": myReport.__to_json__(None), "type": "fwjr" } fwjrdatabase.queue(fwjrDocument, timestamp=True) fwjrdatabase.commit() return
def setUp(self): """ _setUp_ Create two subscriptions: One that contains a single file and one that contains multiple files. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.couchUrl = os.environ["COUCHURL"] self.couchDBName = "acdc_event_based_t" self.testInit.setupCouch(self.couchDBName, "GroupUser", "ACDC") self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False) couchSever = CouchServer(dburl=self.couchUrl) self.couchDB = couchSever.connectDatabase(self.couchDBName) self.populateWMBS() self.performanceParams = { 'timePerEvent': 12, 'memoryRequirement': 2300, 'sizePerEvent': 400 } return