def __init__(self, config):
"""
Initialise class members
"""
#Need a better way to test this without turning off this next line
BaseWorkerThread.__init__(self)
#logging.basicConfig(format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s',datefmt = '%m-%d %H:%M')
#self.logger = logging.getLogger()
# self.logger is set up by the BaseWorkerThread, we just set it's level
self.config = config.AsyncTransfer
try:
self.logger.setLevel(self.config.log_level)
except:
import logging
self.logger = logging.getLogger()
self.logger.setLevel(self.config.log_level)
self.logger.debug('Configuration loaded')
server = CouchServer(dburl=self.config.couch_instance,
ckey=self.config.opsProxy,
cert=self.config.opsProxy)
self.db = server.connectDatabase(self.config.files_database)
config_server = CouchServer(dburl=self.config.config_couch_instance)
self.config_db = config_server.connectDatabase(
self.config.config_database)
self.logger.debug('Connected to CouchDB')
self.pool = Pool(processes=self.config.pool_size)
try:
self.phedex = PhEDEx(responseType='xml')
except Exception, e:
self.logger.exception('PhEDEx exception: %s' % e)
def __init__(self, config):
BaseWorkerThread.__init__(self)
self.config = config.Statistics
try:
self.logger.setLevel(self.config.log_level)
except:
import logging
self.logger = logging.getLogger()
self.logger.setLevel(self.config.log_level)
self.logger.debug('Configuration loaded')
server = CouchServer(dburl=self.config.couch_instance,
ckey=self.config.opsProxy,
cert=self.config.opsProxy)
self.db = server.connectDatabase(self.config.files_database)
config_server = CouchServer(dburl=self.config.config_couch_instance,
ckey=self.config.opsProxy,
cert=self.config.opsProxy)
self.config_db = config_server.connectDatabase(
self.config.config_database)
self.mon_db = server.connectDatabase(self.config.mon_database)
self.logger.debug('Connected to CouchDB')
statserver = CouchServer(self.config.couch_statinstance)
self.statdb = statserver.connectDatabase(
self.config.statitics_database)
self.logger.debug('Connected to Stat CouchDB')
self.iteration_docs = []
self.exptime = None
def __init__(self, couchURL, statSummaryDB, summaryLevel):
# set the connection for local couchDB call
self.couchURL = couchURL
self.couchURLBase, self.dbName = splitCouchServiceURL(couchURL)
self.jobCouchDB = CouchServer(self.couchURLBase).connectDatabase(self.dbName + "/jobs", False)
self.fwjrsCouchDB = CouchServer(self.couchURLBase).connectDatabase(self.dbName + "/fwjrs", False)
#TODO: remove the hard coded name (wma_summarydb)
self.summaryStatsDB = CouchServer(self.couchURLBase).connectDatabase(statSummaryDB, False)
self.summaryLevel = summaryLevel
def __init__(self, couchURL, summaryLevel):
# set the connection for local couchDB call
self.couchURL = couchURL
self.couchURLBase, self.dbName = splitCouchServiceURL(couchURL)
self.jobCouchDB = CouchServer(self.couchURLBase).connectDatabase(
self.dbName + "/jobs", False)
self.fwjrsCouchDB = CouchServer(self.couchURLBase).connectDatabase(
self.dbName + "/fwjrs", False)
self.summaryLevel = summaryLevel
def __init__(self, couchURL, statSummaryDB, summaryLevel):
# set the connection for local couchDB call
self.couchURL = couchURL
self.couchURLBase, self.dbName = splitCouchServiceURL(couchURL)
self.jobCouchDB = CouchServer(self.couchURLBase).connectDatabase(self.dbName + "/jobs", False)
fwjrDBname = "%s/fwjrs" % self.dbName
self.fwjrAPI = FWJRDBAPI(self.couchURLBase, fwjrDBname)
self.summaryStatsDB = CouchServer(self.couchURLBase).connectDatabase(statSummaryDB, False)
self.summaryLevel = summaryLevel
def __init__(self, couchURL, dbName = None):
couchURL = sanitizeURL(couchURL)['url']
# set the connection for local couchDB call
if dbName:
self.couchURL = couchURL
self.dbName = dbName
else:
self.couchURL, self.dbName = splitCouchServiceURL(couchURL)
self.couchServer = CouchServer(self.couchURL)
self.couchDB = CouchServer(self.couchURL).connectDatabase(self.dbName, False)
def __init__(self, config):
BaseDaemon.__init__(self, config, 'Analytics')
server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
self.db = server.connectDatabase(self.config.files_database)
self.logger.debug('Connected to local couchDB')
config_server = CouchServer(dburl=self.config.config_couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
self.config_db = config_server.connectDatabase(self.config.config_database)
self.amq_auth_file = self.config.amq_auth_file
monitoring_server = CouchServer(dburl=self.config.couch_user_monitoring_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
self.monitoring_db = monitoring_server.connectDatabase(self.config.user_monitoring_db)
self.logger.debug('Connected to user_monitoring_db in couchDB')
def setUp(self):
myThread = threading.currentThread()
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
self.databaseName = "couchapp_t_0"
self.testInit.setupCouch(self.databaseName, "WorkloadSummary")
self.testInit.setupCouch("%s/jobs" % self.databaseName, "JobDump")
self.testInit.setupCouch("%s/fwjrs" % self.databaseName, "FWJRDump")
# Setup config for couch connections
config = self.testInit.getConfiguration()
config.section_("JobStateMachine")
config.JobStateMachine.couchurl = os.getenv("COUCHURL",
"cmssrv52.fnal.gov:5984")
config.JobStateMachine.couchDBName = self.databaseName
# Create couch server and connect to databases
self.couchdb = CouchServer(config.JobStateMachine.couchurl)
self.jobsdatabase = self.couchdb.connectDatabase(
"%s/jobs" % config.JobStateMachine.couchDBName)
self.fwjrdatabase = self.couchdb.connectDatabase(
"%s/fwjrs" % config.JobStateMachine.couchDBName)
# Create changeState
self.changeState = ChangeState(config)
self.config = config
# Create testDir
self.testDir = self.testInit.generateWorkDir()
return
def setUp(self):
"""
_setUp_
Initialize the database and couch.
"""
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setupCouch("promptreco_t", "ConfigCache")
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
couchServer = CouchServer(os.environ["COUCHURL"])
self.configDatabase = couchServer.connectDatabase("promptreco_t")
self.testDir = self.testInit.generateWorkDir()
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.listTasksByWorkflow = self.daoFactory(classname="Workflow.LoadFromName")
self.listFilesets = self.daoFactory(classname="Fileset.List")
self.listSubsMapping = self.daoFactory(classname="Subscriptions.ListSubsAndFilesetsFromWorkflow")
self.promptSkim = None
if PY3:
self.assertItemsEqual = self.assertCountEqual
return
def __init__(self, cfg_dict={}, reg_info={}):
"""
Initialise the regsvc for this component,
"""
try:
config_dict = {
'server': 'https://cmsweb.cern.ch/',
'database': 'registration',
'cacheduration': 1,
}
config_dict.update(cfg_dict)
self.server = CouchServer(config_dict['server'])
self.db = self.server.connectDatabase(config_dict['database'])
if 'location' not in reg_info.keys():
raise KeyError('Registration needs a location in its reg_info')
self.location_hash = str(reg_info['location'].__hash__())
reg_info['_id'] = self.location_hash
reg_info['#config_hash'] = hash(str(reg_info))
push_cfg = True
if self.db.documentExists(self.location_hash):
# If the doc exists, check that the configuration hasn't changed
doc = self.db.document(self.location_hash)
push_cfg = doc['#config_hash'] != reg_info['#config_hash']
reg_info['_rev'] = doc['_rev']
if push_cfg:
self.db.commitOne(reg_info)
except:
# Don't want to raise anything here
# TODO: but should probably log...
pass
self.report()
def __init__(self, config):
# configuration values:
# 'uri' attribute (URL of the REST server and resource name)
# in case of CouchDB, the resource name is the database name
# http://servername:port/databaseName
self.config = config
logging.info("Instantiating ...")
# the class currently relies only on 1 REST server possibility - the
# CouchDB server. as explained above, .database will be replaced by
# .connection if both a generic REST server as well as CouchDB are to
# be talked to
split = self.config.uri.rfind('/')
dbName = self.config.uri[split +
1:] # get last item of URI - database name
url = self.config.uri[:split]
# as opposed to CouchSink, here it's assumed the resource (the database name)
# does exist, fail here otherwise
# this check / rest of the constructed may be revised for
# general REST server
server = CouchServer(url)
databases = server.listDatabases()
# there needs to be this database created upfront and also
# couchapp associated with it installed, if it's there, fail
if dbName not in databases:
raise Exception("REST URI: %s (DB name: %s) does not exist." %
(self.config.uri, dbName))
self._database = Database(dbName, url)
logging.info("Initialized.")
def saveCouch(self, couchUrl, couchDBName, metadata=None):
""" Save this spec in CouchDB. Returns URL """
from WMCore.Database.CMSCouch import CouchServer, CouchInternalServerError
metadata = metadata or {}
server = CouchServer(couchUrl)
database = server.connectDatabase(couchDBName)
name = self.name()
uri = '/%s/%s' % (couchDBName, name)
specuri = uri + '/spec'
if not database.documentExists(name):
self.setSpecUrl(couchUrl + specuri)
doc = database.put(uri, data=metadata, contentType='application/json')
# doc = database.commitOne(self.name(), metadata)
rev = doc['rev']
else:
# doc = database.get(uri+'?revs=true')
doc = database.document(name)
rev = doc['_rev']
# specuriwrev = specuri + '?rev=%s' % rev
# FIXME: once both central services and WMAgent are in Py3, we can remove protocol=0
workloadString = pickle.dumps(self.data, protocol=0)
# result = database.put(specuriwrev, workloadString, contentType='application/text')
retval = database.addAttachment(name, rev, workloadString, 'spec')
if retval.get('ok', False) is not True:
msg = "Failed to save a spec attachment in CouchDB for %s" % name
raise CouchInternalServerError(msg, data=None, result=retval)
url = couchUrl + specuri
return url
def setUp(self):
"""
_setUp_
Setup couchdb and the test environment
"""
super(ResubmitBlockTest, self).setUp()
self.group = 'unknown'
self.user = '******'
# Set external test helpers
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setupCouch("resubmitblock_t", "ACDC", "GroupUser")
# Define test environment
self.couchUrl = os.environ["COUCHURL"]
self.acdcDBName = 'resubmitblock_t'
self.validLocations = [
'T2_US_Nebraska', 'T1_US_FNAL_Disk', 'T1_UK_RAL_Disk'
]
self.siteWhitelist = ['T2_XX_SiteA']
cric = CRIC()
# Convert phedex node name to a valid processing site name
self.PSNs = cric.PNNstoPSNs(self.validLocations)
self.workflowName = 'dballest_ReReco_workflow'
couchServer = CouchServer(dburl=self.couchUrl)
self.acdcDB = couchServer.connectDatabase(self.acdcDBName,
create=False)
user = makeUser(self.group, '*****@*****.**', self.couchUrl,
self.acdcDBName)
user.create()
return
def __init__(self,
db_url,
db_name='workqueue',
inbox_name=None,
parentQueue=None,
queueUrl=None,
logger=None):
if logger:
self.logger = logger
else:
import logging
self.logger = logging
if inbox_name is None:
inbox_name = "%s_inbox" % db_name
self.server = CouchServer(db_url)
self.parentCouchUrlWithAuth = parentQueue
if parentQueue:
self.parentCouchUrl = sanitizeURL(parentQueue)['url']
else:
self.parentCouchUrl = None
self.db = self.server.connectDatabase(db_name,
create=False,
size=10000)
self.hostWithAuth = db_url
self.inbox = self.server.connectDatabase(inbox_name,
create=False,
size=10000)
self.queueUrl = sanitizeURL(queueUrl
or (db_url + '/' + db_name))['url']
self.eleKey = 'WMCore.WorkQueue.DataStructs.WorkQueueElement.WorkQueueElement'
def main(options):
url, dbName = splitCouchServiceURL(options.dburl)
db = CouchServer(url).connectDatabase(dbName)
reqmgr_requests = generate_reqmgr_requests(options.requests)
agent_requests = generate_agent_requests(options.requests,
options.iterations)
if options.add_couchapp:
installCouchApp(url, dbName, "WMStats", options.couchapp_base)
if options.add_reqmgr_data:
for req in reqmgr_requests:
db.queue(req)
db.commit()
print("Added %s reqmgr requests" % len(reqmgr_requests))
if options.add_agent_data:
for req in agent_requests:
db.queue(req)
jobDocs = generate_jobsummary(req['workflow'])
for job in jobDocs:
db.queue(job)
db.commit()
print("Added %s agent requests" % len(agent_requests))
print("Added %s job Docs" % (len(agent_requests) * len(jobDocs)))
def setUp(self):
"""
_setUp_
"""
self.transitions = Transitions()
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setupCouch("changestate_t/jobs", "JobDump")
self.testInit.setupCouch("changestate_t/fwjrs", "FWJRDump")
self.testInit.setupCouch("job_summary", "WMStats")
self.testInit.setSchema(customModules = ["WMCore.WMBS"],
useDefault = False)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
couchurl = os.getenv("COUCHURL")
self.couchServer = CouchServer(dburl = couchurl)
self.config = self.testInit.getConfiguration()
self.taskName = "/TestWorkflow/ReReco1"
self.specGen = WMSpecGenerator()
self.specUrl = self.specGen.createProcessingSpec("TestWorkflow", returnType="file")
return
def _commonInit(self, couchURL, couchapp):
"""
setting up comon variables for inherited class.
inherited class should call this in their init function
"""
if isinstance(couchURL, Database):
self.couchDB = couchURL
self.couchURL = self.couchDB['host']
self.dbName = self.couchDB.name
self.couchServer = CouchServer(self.couchURL)
else:
self.couchURL, self.dbName = splitCouchServiceURL(couchURL)
self.couchServer = CouchServer(self.couchURL)
self.couchDB = self.couchServer.connectDatabase(self.dbName, False)
self.couchapp = couchapp
self.defaultStale = {"stale": "update_after"}
def __init__(self,
db_url,
db_name='workqueue',
inbox_name='workqueue_inbox',
parentQueue=None,
queueUrl=None,
logger=None):
if logger:
self.logger = logger
else:
import logging
self.logger = logging
self.server = CouchServer(db_url)
self.parentCouchUrlWithAuth = parentQueue
if parentQueue:
self.parentCouchUrl = sanitizeURL(parentQueue)['url']
else:
self.parentCouchUrl = None
self.db = self.server.connectDatabase(db_name,
create=False,
size=10000)
self.hostWithAuth = db_url
self.inbox = self.server.connectDatabase(inbox_name,
create=False,
size=10000)
self.queueUrl = sanitizeURL(queueUrl
or (db_url + '/' + db_name))['url']
def __init__(self, config):
BaseWorkerThread.__init__(self)
self.config = config.FilesCleaner
self.logger.debug('Configuration loaded')
try:
self.logger.setLevel(self.config.log_level)
except:
import logging
self.logger = logging.getLogger()
self.logger.setLevel(self.config.log_level)
self.logger.debug('Configuration loaded')
config_server = CouchServer(dburl=self.config.config_couch_instance)
self.config_db = config_server.connectDatabase(
self.config.config_database)
self.logger.debug('Connected to files DB')
self.phedex = PhEDEx(responseType='xml')
self.log_dir = '%s/logs/%s/%s/%s' % ( self.config.componentDir, \
str(datetime.datetime.now().month), str(datetime.datetime.now().year), "Ops")
try:
os.makedirs(self.log_dir)
except OSError, e:
if e.errno == errno.EEXIST:
pass
else:
self.logger.error('Unknown error in mkdir' % e.errno)
raise
def __init__(self, db_url, db_name, identifier, thread_name, **kwds):
self.db_url = db_url
self.server = CouchServer(db_url)
self.db_name = db_name
self.dbid = identifier
self.thread_name = thread_name
self.agent = kwds.get('agent', 0)
create = kwds.get('create', False)
size = kwds.get('size', 10000)
self.db = self.server.connectDatabase(db_name,
create=create,
size=size)
self.design = kwds.get('design', 'LogDB') # name of design document
self.view = kwds.get('view',
'requests') # name of view to look-up requests
self.tsview = kwds.get('tsview',
'tstamp') # name of tsview to look-up requests
if create:
uri = '/%s/_design/%s' % (db_name, self.design)
data = design_doc()
try:
# insert design doc, if fails due to conflict continue
# conflict may happen due to concurrent client connection who
# created first this doc
self.db.put(uri, data)
except CouchConflictError:
pass
def testTimestampAccounting(self):
"""
_testTimestampAccounting_
Check the correct functioning of the timestamp view in the ACDC
couchapp and the function to remove old filesets.
"""
self.populateCouchDB()
svc = CouchService(url=self.testInit.couchUrl,
database=self.testInit.couchDbName)
currentTime = time.time()
database = CouchServer(self.testInit.couchUrl).connectDatabase(
self.testInit.couchDbName)
results = database.loadView("ACDC", "byTimestamp",
{"endkey": currentTime})
self.assertEqual(len(results["rows"]), 4)
results = database.loadView("ACDC", "byTimestamp",
{"endkey": currentTime - 2})
self.assertEqual(len(results["rows"]), 3)
results = database.loadView("ACDC", "byTimestamp",
{"endkey": currentTime - 3})
self.assertEqual(len(results["rows"]), 2)
results = database.loadView("ACDC", "byTimestamp",
{"endkey": currentTime - 4})
self.assertEqual(len(results["rows"]), 1)
results = database.loadView("ACDC", "byTimestamp",
{"endkey": currentTime - 5})
self.assertEqual(len(results["rows"]), 0)
svc.removeOldFilesets(0)
results = database.loadView("ACDC", "byTimestamp",
{"endkey": currentTime})
self.assertEqual(len(results["rows"]), 0)
return
def __init__(self, config, couchDbName=None):
WMObject.__init__(self, config)
WMConnectionBase.__init__(self, "WMCore.WMBS")
if couchDbName == None:
self.dbname = getattr(self.config.JobStateMachine, "couchDBName")
else:
self.dbname = couchDbName
try:
self.couchdb = CouchServer(self.config.JobStateMachine.couchurl)
self.jobsdatabase = self.couchdb.connectDatabase("%s/jobs" %
self.dbname,
size=250)
self.fwjrdatabase = self.couchdb.connectDatabase("%s/fwjrs" %
self.dbname,
size=250)
self.jsumdatabase = self.couchdb.connectDatabase(getattr(
self.config.JobStateMachine, 'jobSummaryDBName'),
size=250)
except Exception, ex:
logging.error("Error connecting to couch: %s" % str(ex))
self.jobsdatabase = None
self.fwjrdatabase = None
self.jsumdatabase = None
def testRemoveByCollectionName(self):
"""
_testRemoveByCollectionName_
Check the function to obliterate all the filesets of a collection
"""
self.populateCouchDB()
svc = CouchService(url=self.testInit.couchUrl,
database=self.testInit.couchDbName)
database = CouchServer(self.testInit.couchUrl).connectDatabase(
self.testInit.couchDbName)
results = database.loadView("ACDC",
"byCollectionName",
keys=["Thunderstruck"])
self.assertTrue(len(results["rows"]) > 0)
svc.removeFilesetsByCollectionName("Thunderstruck")
results = database.loadView("ACDC",
"byCollectionName",
keys=["Thunderstruck"])
self.assertEqual(len(results["rows"]), 0)
results = database.loadView("ACDC",
"byCollectionName",
keys=["Struckthunder"])
self.assertTrue(len(results["rows"]) > 0)
svc.removeFilesetsByCollectionName("Struckthunder")
results = database.loadView("ACDC",
"byCollectionName",
keys=["Struckthunder"])
self.assertEqual(len(results["rows"]), 0)
return
def __init__(self, config, couchDbName=None):
WMObject.__init__(self, config)
WMConnectionBase.__init__(self, "WMCore.WMBS")
if couchDbName is None:
self.dbname = getattr(self.config.JobStateMachine, "couchDBName")
else:
self.dbname = couchDbName
self.jobsdatabase = None
self.fwjrdatabase = None
self.jsumdatabase = None
self.statsumdatabase = None
self.couchdb = CouchServer(self.config.JobStateMachine.couchurl)
self._connectDatabases()
try:
self.dashboardReporter = DashboardReporter(config)
except Exception as ex:
logging.error("Error setting up the dashboard reporter: %s", str(ex))
raise
self.getCouchDAO = self.daofactory("Jobs.GetCouchID")
self.setCouchDAO = self.daofactory("Jobs.SetCouchID")
self.incrementRetryDAO = self.daofactory("Jobs.IncrementRetry")
self.workflowTaskDAO = self.daofactory("Jobs.GetWorkflowTask")
self.jobTypeDAO = self.daofactory("Jobs.GetType")
self.updateLocationDAO = self.daofactory("Jobs.UpdateLocation")
self.getWorkflowSpecDAO = self.daofactory("Workflow.GetSpecAndNameFromTask")
self.maxUploadedInputFiles = getattr(self.config.JobStateMachine, 'maxFWJRInputFiles', 1000)
self.workloadCache = {}
return
def resubmitCouchPublication(self, asourl, asodb, proxy, taskname):
"""
Resubmit failed publications by resetting the publication
status in the CouchDB documents.
"""
server = CouchServer(dburl=asourl, ckey=proxy, cert=proxy)
try:
database = server.connectDatabase(asodb)
except Exception as ex:
msg = "Error while trying to connect to CouchDB: %s" % (str(ex))
raise Exception(msg)
try:
failedPublications = database.loadView('DBSPublisher', 'PublicationFailedByWorkflow', {'reduce': False, 'startkey': [taskname], 'endkey': [taskname, {}]})['rows']
except Exception as ex:
msg = "Error while trying to load view 'DBSPublisher.PublicationFailedByWorkflow' from CouchDB: %s" % (str(ex))
raise Exception(msg)
msg = "There are %d failed publications to resubmit: %s" % (len(failedPublications), failedPublications)
self.logger.info(msg)
for doc in failedPublications:
docid = doc['id']
if doc['key'][0] != taskname: # this should never happen...
msg = "Skipping document %s as it seems to correspond to another task: %s" % (docid, doc['key'][0])
self.logger.warning(msg)
continue
data = {'last_update': time.time(),
'retry': str(datetime.datetime.now()),
'publication_state': 'not_published',
}
try:
database.updateDocument(docid, 'DBSPublisher', 'updateFile', data)
self.logger.info("updating document %s ", docid)
except Exception as ex:
self.logger.error("Error updating document %s in CouchDB: %s", docid, str(ex))
return
def setUp(self):
self.couchURL = os.getenv("COUCHURL")
self.server = CouchServer(self.couchURL)
# Kill off any databases left over from previous runs
for db in [
db for db in self.server.listDatabases()
if db.startswith('rotdb_unittest_')
]:
try:
self.server.deleteDatabase(db)
except:
pass
# Create a database, drop an existing one first
testname = self.id().split('.')[-1].lower()
self.dbname = 'rotdb_unittest_%s' % testname
self.arcname = 'rotdb_unittest_%s_archive' % testname
self.seedname = 'rotdb_unittest_%s_seedcfg' % testname
# set a long value for times, tests do operations explicitly
self.timing = {
'archive': timedelta(seconds=1),
'expire': timedelta(seconds=2)
}
self.db = RotatingDatabase(dbname=self.dbname,
url=self.couchURL,
archivename=self.arcname,
timing=self.timing)
def setUp(self):
"""
_setUp_
Initialize the database and couch.
"""
super(MonteCarloTest, self).setUp()
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setupCouch(TEST_DB_NAME, "ConfigCache")
self.testInit.setSchema(customModules=["WMCore.WMBS"], useDefault=False)
self.testInit.generateWorkDir()
couchServer = CouchServer(os.environ["COUCHURL"])
self.configDatabase = couchServer.connectDatabase(TEST_DB_NAME)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.listTasksByWorkflow = self.daoFactory(classname="Workflow.LoadFromName")
self.listFilesets = self.daoFactory(classname="Fileset.List")
self.listSubsMapping = self.daoFactory(classname="Subscriptions.ListSubsAndFilesetsFromWorkflow")
return
def __init__(self, config):
BaseWorkerThread.__init__(self)
self.config = config.AsyncTransfer
# self.logger is set up by the BaseWorkerThread, we just set it's level
try:
self.logger.setLevel(self.config.log_level)
except:
import logging
self.logger = logging.getLogger()
self.logger.setLevel(self.config.log_level)
self.logger.debug('Configuration loaded')
# Set up a factory for loading plugins
self.factory = WMFactory(self.config.pluginDir, namespace = self.config.pluginDir)
# Asynch db
server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
self.db = server.connectDatabase(self.config.files_database)
self.logger.debug('Connected to CouchDB')
return
def testMultiCoreReport(self):
"""
_testMultiCoreReport_
Verify that multicore reports can be json encoded and uploaded to couch.
"""
couchdb = CouchServer(os.environ["COUCHURL"])
fwjrdatabase = couchdb.connectDatabase("report_t/fwjrs")
self.mcPath = os.path.join(
WMCore.WMBase.getTestBase(),
"WMCore_t/FwkJobReport_t/MulticoreReport.pkl")
myReport = Report()
myReport.unpersist(self.mcPath)
fwjrDocument = {
"_id": "303-0",
"jobid": 303,
"retrycount": 0,
"fwjr": myReport.__to_json__(None),
"type": "fwjr"
}
fwjrdatabase.queue(fwjrDocument, timestamp=True)
fwjrdatabase.commit()
return
def setUp(self):
"""
_setUp_
Create two subscriptions: One that contains a single file and one that
contains multiple files.
"""
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.couchUrl = os.environ["COUCHURL"]
self.couchDBName = "acdc_event_based_t"
self.testInit.setupCouch(self.couchDBName, "GroupUser", "ACDC")
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
couchSever = CouchServer(dburl=self.couchUrl)
self.couchDB = couchSever.connectDatabase(self.couchDBName)
self.populateWMBS()
self.performanceParams = {
'timePerEvent': 12,
'memoryRequirement': 2300,
'sizePerEvent': 400
}
return
