class DBSUploadTest(unittest.TestCase):
"""
TestCase for DBSUpload module
"""
def setUp(self):
"""
_setUp_
setUp function for unittest
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer"],
useDefault = False)
self.testDir = self.testInit.generateWorkDir(deleteOnDestruction = False)
self.configFile = EmulatorSetup.setupWMAgentConfig()
myThread = threading.currentThread()
self.bufferFactory = DAOFactory(package = "WMComponent.DBSBuffer.Database",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.buffer3Factory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = self.bufferFactory(classname = "DBSBufferFiles.AddLocation")
locationAction.execute(siteName = "se1.cern.ch")
locationAction.execute(siteName = "se1.fnal.gov")
locationAction.execute(siteName = "malpaquet")
self.dbsUrl = "https://*****:*****@attr("integration")
def testBasicUpload(self):
"""
_testBasicUpload_
Verify that we can successfully upload to DBS3. Also verify that the
uploader correctly handles files parentage when uploading.
"""
self.dbsApi = DbsApi(url = self.dbsUrl)
config = self.getConfig()
dbsUploader = DBSUploadPoller(config = config)
# First test verifies that uploader will poll and then not do anything
# as the database is empty.
dbsUploader.algorithm()
acqEra = "Summer%s" % (int(time.time()))
parentFiles = self.createParentFiles(acqEra)
# The algorithm needs to be run twice. On the first iteration it will
# create all the blocks and upload one. On the second iteration it will
# timeout and upload the second block.
dbsUploader.algorithm()
time.sleep(5)
dbsUploader.algorithm()
time.sleep(5)
# Verify the files made it into DBS3.
self.verifyData(parentFiles[0]["datasetPath"], parentFiles)
# Inject some more parent files and some child files into DBSBuffer.
# Run the uploader twice, only the parent files should be added to DBS3.
(moreParentFiles, childFiles) = \
self.createFilesWithChildren(parentFiles, acqEra)
dbsUploader.algorithm()
time.sleep(5)
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(parentFiles[0]["datasetPath"],
parentFiles + moreParentFiles)
# Run the uploader another two times to upload the child files. Verify
# that the child files were uploaded.
dbsUploader.algorithm()
time.sleep(5)
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(childFiles[0]["datasetPath"], childFiles)
return
@attr("integration")
def testDualUpload(self):
"""
_testDualUpload_
Verify that the dual upload mode works correctly.
"""
self.dbsApi = DbsApi(url = self.dbsUrl)
config = self.getConfig(dbs3UploadOnly = True)
dbsUploader = DBSUploadPoller(config = config)
dbsUtil = DBSBufferUtil()
# First test verifies that uploader will poll and then not do anything
# as the database is empty.
dbsUploader.algorithm()
acqEra = "Summer%s" % (int(time.time()))
parentFiles = self.createParentFiles(acqEra)
(moreParentFiles, childFiles) = \
self.createFilesWithChildren(parentFiles, acqEra)
allFiles = parentFiles + moreParentFiles
allBlocks = []
for i in range(4):
blockName = parentFiles[0]["datasetPath"] + "#" + makeUUID()
dbsBlock = DBSBlock(blockName, "malpaquet", 1)
dbsBlock.status = "Open"
dbsUtil.createBlocks([dbsBlock])
for file in allFiles[i * 5 : (i * 5) + 5]:
dbsBlock.addFile(file)
dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]})
if i < 2:
dbsBlock.status = "InDBS"
dbsUtil.updateBlocks([dbsBlock])
dbsUtil.updateFileStatus([dbsBlock], "InDBS")
allBlocks.append(dbsBlock)
blockName = childFiles[0]["datasetPath"] + "#" + makeUUID()
dbsBlock = DBSBlock(blockName, "malpaquet", 1)
dbsBlock.status = "InDBS"
dbsUtil.createBlocks([dbsBlock])
for file in childFiles:
dbsBlock.addFile(file)
dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]})
dbsUtil.updateFileStatus([dbsBlock], "InDBS")
dbsUploader.algorithm()
time.sleep(5)
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(parentFiles[0]["datasetPath"], parentFiles)
# Change the status of the rest of the parent blocks so we can upload
# them and the children.
for dbsBlock in allBlocks:
dbsBlock.status = "InDBS"
dbsUtil.updateBlocks([dbsBlock])
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(parentFiles[0]["datasetPath"], parentFiles + moreParentFiles)
# Run the uploader one more time to upload the children.
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(childFiles[0]["datasetPath"], childFiles)
return
def testCloseSettingsPerWorkflow(self):
"""
_testCloseSettingsPerWorkflow_
Test the block closing mechanics in the DBS3 uploader,
this uses a fake dbs api to avoid reliance on external services.
"""
# Signal trapExit that we are a friend
os.environ["DONT_TRAP_EXIT"] = "True"
try:
# Monkey patch the imports of DbsApi
from WMComponent.DBS3Buffer import DBSUploadPoller as MockDBSUploadPoller
MockDBSUploadPoller.DbsApi = MockDbsApi
# Set the poller and the dbsUtil for verification
myThread = threading.currentThread()
(_, dbsFilePath) = mkstemp(dir = self.testDir)
self.dbsUrl = dbsFilePath
config = self.getConfig()
dbsUploader = MockDBSUploadPoller.DBSUploadPoller(config = config)
dbsUtil = DBSBufferUtil()
# First test is event based limits and timeout with no new files.
# Set the files and workflow
acqEra = "TropicalSeason%s" % (int(time.time()))
workflowName = 'TestWorkload%s' % (int(time.time()))
taskPath = '/%s/TestProcessing' % workflowName
self.injectWorkflow(workflowName, taskPath,
MaxWaitTime = 2, MaxFiles = 100,
MaxEvents = 150)
self.createParentFiles(acqEra, nFiles = 20,
workflowName = workflowName,
taskPath = taskPath)
# The algorithm needs to be run twice. On the first iteration it will
# create all the blocks and upload one with less than 150 events.
# On the second iteration the second block is uploaded.
dbsUploader.algorithm()
dbsUploader.checkBlocks()
openBlocks = dbsUtil.findOpenBlocks()
self.assertEqual(len(openBlocks), 1)
globalFiles = myThread.dbi.processData("SELECT id FROM dbsbuffer_file WHERE status = 'InDBS'")[0].fetchall()
notUploadedFiles = myThread.dbi.processData("SELECT id FROM dbsbuffer_file WHERE status = 'NOTUPLOADED'")[0].fetchall()
self.assertEqual(len(globalFiles), 14)
self.assertEqual(len(notUploadedFiles), 6)
# Check the fake DBS for data
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 2)
for block in fakeDBSInfo:
self.assertTrue('block_events' not in block['block'])
self.assertEqual(block['block']['file_count'], 7)
self.assertEqual(block['block']['open_for_writing'], 0)
self.assertTrue('close_settings' not in block)
time.sleep(3)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
openBlocks = dbsUtil.findOpenBlocks()
self.assertEqual(len(openBlocks), 0)
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 3)
for block in fakeDBSInfo:
if block['block']['file_count'] != 6:
self.assertEqual(block['block']['file_count'], 7)
self.assertTrue('block_events' not in block['block'])
self.assertEqual(block['block']['open_for_writing'], 0)
self.assertTrue('close_settings' not in block)
# Now check the limit by size and timeout with new files
acqEra = "TropicalSeason%s" % (int(time.time()))
workflowName = 'TestWorkload%s' % (int(time.time()))
taskPath = '/%s/TestProcessing' % workflowName
self.injectWorkflow(workflowName, taskPath,
MaxWaitTime = 2, MaxFiles = 5,
MaxEvents = 200000000)
self.createParentFiles(acqEra, nFiles = 16,
workflowName = workflowName,
taskPath = taskPath)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
openBlocks = dbsUtil.findOpenBlocks()
self.assertEqual(len(openBlocks), 1)
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 6)
for block in fakeDBSInfo:
if acqEra in block['block']['block_name']:
self.assertEqual(block['block']['file_count'], 5)
self.assertTrue('block_events' not in block['block'])
self.assertTrue('close_settings' not in block)
self.assertEqual(block['block']['open_for_writing'], 0)
# Put more files, they will go into the same block and then it will be closed
# after timeout
time.sleep(3)
self.createParentFiles(acqEra, nFiles = 3,
workflowName = workflowName,
taskPath = taskPath)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
openBlocks = dbsUtil.findOpenBlocks()
self.assertEqual(len(openBlocks), 0)
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 7)
for block in fakeDBSInfo:
if acqEra in block['block']['block_name']:
if block['block']['file_count'] < 5:
self.assertEqual(block['block']['file_count'], 4)
else:
self.assertEqual(block['block']['file_count'], 5)
self.assertTrue('block_events' not in block['block'])
self.assertEqual(block['block']['open_for_writing'], 0)
self.assertTrue('close_settings' not in block)
# Finally test size limits
acqEra = "TropicalSeason%s" % (int(time.time()))
workflowName = 'TestWorkload%s' % (int(time.time()))
taskPath = '/%s/TestProcessing' % workflowName
self.injectWorkflow(workflowName, taskPath,
MaxWaitTime = 1, MaxFiles = 500,
MaxEvents = 200000000, MaxSize = 2048)
self.createParentFiles(acqEra, nFiles = 7,
workflowName = workflowName,
taskPath = taskPath)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
time.sleep(2)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
self.assertEqual(len(openBlocks), 0)
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 11)
for block in fakeDBSInfo:
if acqEra in block['block']['block_name']:
if block['block']['file_count'] != 1:
self.assertEqual(block['block']['block_size'], 2048)
self.assertEqual(block['block']['file_count'], 2)
self.assertTrue('block_events' not in block['block'])
self.assertEqual(block['block']['open_for_writing'], 0)
self.assertTrue('close_settings' not in block)
except:
self.fail("We failed at some point in the test")
finally:
# We don't trust anyone else with _exit
del os.environ["DONT_TRAP_EXIT"]
return
class ThreadPoolTest(unittest.TestCase):
"""
_ThreadPool_t_
Unit tests for threadpool
"""
_nrOfThreads = 10
_nrOfPools = 5
def setUp(self):
"make a logger instance and create tables"
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema()
def tearDown(self):
"""
Deletion of database
"""
# FIXME: this might not work if your not using socket.
self.testInit.clearDatabase()
def testA(self):
"""
__testSubscribe__
Test subscription of a component.
"""
raise nose.SkipTest
myThread = threading.currentThread()
# create a 'fake' component that contains a arg dictionary.
component = Dummy()
# load default parameters.
config = self.testInit.getConfiguration()
# normally assigned by the harness of the test component.
config.Agent.componentName = "TestComponent"
component.config = config
threadPools = []
for i in xrange(0, ThreadPoolTest._nrOfPools):
threadPool = ThreadPool("WMCore.ThreadPool.ThreadSlave", \
component, 'MyPool_'+str(i), ThreadPoolTest._nrOfThreads)
threadPools.append(threadPool)
# this is how you would use the threadpool. The threadpool retrieves
# events/payloads from the message service. If a thread is available
# it is dispatched, otherwise it is stored in the trheadpool.
# make the number of tasks bigger than number of threads to tesT
# the persistent queue.
for i in xrange(0, ThreadPoolTest._nrOfThreads * 10):
event = 'eventNr_' + str(i)
payload = 'payloadNr_' + str(i)
# normally you would have different events per threadpool and
# even different objects per pool. the payload part will be
# pickled into the database enabling flexibility in passing
# information.
for j in xrange(0, ThreadPoolTest._nrOfPools):
threadPools[j].enqueue(event, \
{'event' : event, 'payload' : payload})
# this commit you want to be in the agent harness, so the message is
# actual removed from the msgService. we can do this as the threadpool
# acts as a dispatcher and is a shortlived action: dispatch to thread
# or queue and tell agent harness it is finished.
finished = False
timeout = 60 # secs
currenttime = 0
while not finished:
print('waiting for threads to finishs. Work left:')
for j in xrange(0, ThreadPoolTest._nrOfPools):
print('pool_' + str(j) + ':' + str(threadPools[j].callQueue))
time.sleep(1)
finished = True
currenttime += 1
if (timeout == currenttime):
raise RuntimeError
for j in xrange(0, ThreadPoolTest._nrOfPools):
if (len(threadPools[j].resultsQueue) <
ThreadPoolTest._nrOfThreads * 10):
finished = False
break
# check if the tables are really empty and all messages
# have been processed.
for j in xrange(0, ThreadPoolTest._nrOfPools):
assert len(threadPools[j].resultsQueue) == \
ThreadPoolTest._nrOfThreads*10
myThread.transaction.begin()
for j in xrange(0, ThreadPoolTest._nrOfPools):
self.assertEqual(threadPools[j].countMessages(), 0)
myThread.transaction.commit()
class ThreadPoolTest(unittest.TestCase):
"""
_ThreadPool_t_
Unit tests for threadpool
"""
_nrOfThreads = 10
_nrOfPools = 5
def setUp(self):
"make a logger instance and create tables"
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema()
def tearDown(self):
"""
Deletion of database
"""
# FIXME: this might not work if your not using socket.
self.testInit.clearDatabase()
def testA(self):
"""
__testSubscribe__
Test subscription of a component.
"""
raise nose.SkipTest
myThread = threading.currentThread()
# create a 'fake' component that contains a arg dictionary.
component = Dummy()
# load default parameters.
config = self.testInit.getConfiguration()
# normally assigned by the harness of the test component.
config.Agent.componentName = "TestComponent"
component.config = config
threadPools = []
for i in xrange(0, ThreadPoolTest._nrOfPools):
threadPool = ThreadPool("WMCore.ThreadPool.ThreadSlave", \
component, 'MyPool_'+str(i), ThreadPoolTest._nrOfThreads)
threadPools.append(threadPool)
# this is how you would use the threadpool. The threadpool retrieves
# events/payloads from the message service. If a thread is available
# it is dispatched, otherwise it is stored in the trheadpool.
# make the number of tasks bigger than number of threads to tesT
# the persistent queue.
for i in xrange(0, ThreadPoolTest._nrOfThreads*10):
event = 'eventNr_'+str(i)
payload = 'payloadNr_'+str(i)
# normally you would have different events per threadpool and
# even different objects per pool. the payload part will be
# pickled into the database enabling flexibility in passing
# information.
for j in xrange(0, ThreadPoolTest._nrOfPools):
threadPools[j].enqueue(event, \
{'event' : event, 'payload' : payload})
# this commit you want to be in the agent harness, so the message is
# actual removed from the msgService. we can do this as the threadpool
# acts as a dispatcher and is a shortlived action: dispatch to thread
# or queue and tell agent harness it is finished.
finished = False
timeout = 60 # secs
currenttime = 0
while not finished:
print('waiting for threads to finishs. Work left:')
for j in xrange(0, ThreadPoolTest._nrOfPools):
print('pool_'+str(j)+ ':' + str(threadPools[j].callQueue))
time.sleep(1)
finished = True
currenttime += 1
if (timeout == currenttime):
raise RuntimeError
for j in xrange(0, ThreadPoolTest._nrOfPools):
if (len(threadPools[j].resultsQueue) < ThreadPoolTest._nrOfThreads*10):
finished = False
break
# check if the tables are really empty and all messages
# have been processed.
for j in xrange(0, ThreadPoolTest._nrOfPools):
assert len(threadPools[j].resultsQueue) == \
ThreadPoolTest._nrOfThreads*10
myThread.transaction.begin()
for j in xrange(0, ThreadPoolTest._nrOfPools):
self.assertEqual( threadPools[j].countMessages() , 0 )
myThread.transaction.commit()
class scaleTestFiller:
"""
_scaleTestFiller_
Initializes the DB and the DBSUploader
On __call__() it creates data and uploads it.
"""
def __init__(self):
"""
__init__
Init the DB
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase = True)
self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer"],
useDefault = False)
self.configFile = EmulatorSetup.setupWMAgentConfig()
myThread = threading.currentThread()
self.bufferFactory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = self.bufferFactory(classname = "DBSBufferFiles.AddLocation")
locationAction.execute(siteName = "se1.cern.ch")
locationAction.execute(siteName = "se1.fnal.gov")
locationAction.execute(siteName = "malpaquet")
config = self.getConfig()
self.dbsUploader = DBSUploadPoller(config = config)
return
def __call__(self):
"""
__call__
Generate some random data
"""
# Generate somewhere between one and a thousand files
name = "ThisIsATest_%s" % (makeUUID())
nFiles = random.randint(10, 2000)
name = name.replace('-', '_')
name = '%s-v0' % name
files = self.getFiles(name = name, nFiles = nFiles)
print("Inserting %i files for dataset %s" % (nFiles * 2, name))
try:
self.dbsUploader.algorithm()
except:
self.dbsUploader.close()
raise
# Repeat just to make sure
try:
self.dbsUploader.algorithm()
except:
self.dbsUploader.close()
raise
return
def getConfig(self):
"""
_getConfig_
This creates the actual config file used by the component
"""
config = self.testInit.getConfiguration()
self.testInit.generateWorkDir(config)
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = 'DBSUpload'
config.Agent.useHeartbeat = False
#Now the CoreDatabase information
#This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("DBS3Upload")
config.DBS3Upload.pollInterval = 10
config.DBS3Upload.logLevel = 'DEBUG'
config.DBS3Upload.DBSBlockMaxFiles = 500
config.DBS3Upload.DBSBlockMaxTime = 600
config.DBS3Upload.DBSBlockMaxSize = 999999999999
config.DBS3Upload.dbsUrl = 'http://cms-xen40.fnal.gov:8787/dbs/prod/global/DBSWriter'
config.DBS3Upload.namespace = 'WMComponent.DBS3Buffer.DBS3Upload'
config.DBS3Upload.componentDir = os.path.join(os.getcwd(), 'Components')
config.DBS3Upload.nProcesses = 1
config.DBS3Upload.dbsWaitTime = 1
return config
def getFiles(self, name, tier = 'RECO', nFiles = 12, site = "malpaquet", nLumis = 1):
"""
Create some quick dummy test files
"""
files = []
for f in range(nFiles):
testFile = DBSBufferFile(lfn = '/data/store/random/random/RANDOM/test/0/%s-%s-%i.root' % (name, site, f), size = 1024,
events = 20, checksums = {'cksum': 1})
testFile.setAlgorithm(appName = name, appVer = "CMSSW_3_1_1",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFile.setDatasetPath("/%s/%s/%s" % (name, name, tier))
lumis = []
for i in range(nLumis):
lumis.append((f * 100000) + i)
testFile.addRun(Run( 1, *lumis))
testFile.setAcquisitionEra(name.split('-')[0])
testFile.setProcessingVer("0")
testFile.setGlobalTag("Weird")
testFile.create()
testFile.setLocation(site)
files.append(testFile)
count = 0
for f in files:
count += 1
testFileChild = DBSBufferFile(lfn = '/data/store/random/random/RANDOM/test/0/%s-%s-%i-child.root' %(name, site, count), size = 1024,
events = 10, checksums = {'cksum': 1})
testFileChild.setAlgorithm(appName = name, appVer = "CMSSW_3_1_1",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileChild.setDatasetPath("/%s/%s_2/RECO" %(name, name))
testFileChild.addRun(Run( 1, *[45]))
testFileChild.create()
testFileChild.setLocation(site)
testFileChild.addParents([f['lfn']])
return files
class PhEDExInjectorPollerTest(unittest.TestCase):
"""
_PhEDExInjectorPollerTest_
Unit tests for the PhEDExInjector. Create some database inside DBSBuffer
and then have the PhEDExInjector upload the data to PhEDEx. Pull the data
back down and verify that everything is complete.
"""
def setUp(self):
"""
_setUp_
Install the DBSBuffer schema into the database and connect to PhEDEx.
"""
self.phedexURL = "https://cmsweb.cern.ch/phedex/datasvc/json/test"
self.dbsURL = "http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet"
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase=True)
self.testInit.setSchema(customModules=["WMComponent.DBS3Buffer"],
useDefault=False)
myThread = threading.currentThread()
daofactory = DAOFactory(package="WMComponent.DBSBuffer.Database",
logger=myThread.logger,
dbinterface=myThread.dbi)
locationAction = daofactory(classname="DBSBufferFiles.AddLocation")
locationAction.execute(siteName="srm-cms.cern.ch")
locationAction.execute(siteName="se.fnal.gov")
self.testFilesA = []
self.testFilesB = []
self.testDatasetA = "/%s/PromptReco-v1/RECO" % makeUUID()
self.testDatasetB = "/%s/CRUZET11-v1/RAW" % makeUUID()
self.phedex = PhEDEx({"endpoint": self.phedexURL}, "json")
return
def tearDown(self):
"""
_tearDown_
Delete the database.
"""
self.testInit.clearDatabase()
def stuffDatabase(self, spec="TestWorkload.pkl"):
"""
_stuffDatabase_
Fill the dbsbuffer with some files and blocks. We'll insert a total
of 5 files spanning two blocks. There will be a total of two datasets
inserted into the datbase.
We'll inject files with the location set as an SE name as well as a
PhEDEx node name as well.
"""
myThread = threading.currentThread()
buffer3Factory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
insertWorkflow = buffer3Factory(classname="InsertWorkflow")
insertWorkflow.execute("BogusRequest", "BogusTask", 0, 0, 0, 0)
checksums = {"adler32": "1234", "cksum": "5678"}
testFileA = DBSBufferFile(lfn=makeUUID(),
size=1024,
events=10,
checksums=checksums,
locations=set(["srm-cms.cern.ch"]))
testFileA.setAlgorithm(appName="cmsRun",
appVer="CMSSW_2_1_8",
appFam="RECO",
psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFileA.setDatasetPath(self.testDatasetA)
testFileA.addRun(Run(2, *[45]))
testFileA.create()
testFileB = DBSBufferFile(lfn=makeUUID(),
size=1024,
events=10,
checksums=checksums,
locations=set(["srm-cms.cern.ch"]))
testFileB.setAlgorithm(appName="cmsRun",
appVer="CMSSW_2_1_8",
appFam="RECO",
psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFileB.setDatasetPath(self.testDatasetA)
testFileB.addRun(Run(2, *[45]))
testFileB.create()
testFileC = DBSBufferFile(lfn=makeUUID(),
size=1024,
events=10,
checksums=checksums,
locations=set(["srm-cms.cern.ch"]))
testFileC.setAlgorithm(appName="cmsRun",
appVer="CMSSW_2_1_8",
appFam="RECO",
psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFileC.setDatasetPath(self.testDatasetA)
testFileC.addRun(Run(2, *[45]))
testFileC.create()
self.testFilesA.append(testFileA)
self.testFilesA.append(testFileB)
self.testFilesA.append(testFileC)
testFileD = DBSBufferFile(lfn=makeUUID(),
size=1024,
events=10,
checksums=checksums,
locations=set(["srm-cms.cern.ch"]))
testFileD.setAlgorithm(appName="cmsRun",
appVer="CMSSW_2_1_8",
appFam="RECO",
psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFileD.setDatasetPath(self.testDatasetB)
testFileD.addRun(Run(2, *[45]))
testFileD.create()
testFileE = DBSBufferFile(lfn=makeUUID(),
size=1024,
events=10,
checksums=checksums,
locations=set(["srm-cms.cern.ch"]))
testFileE.setAlgorithm(appName="cmsRun",
appVer="CMSSW_2_1_8",
appFam="RECO",
psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFileE.setDatasetPath(self.testDatasetB)
testFileE.addRun(Run(2, *[45]))
testFileE.create()
self.testFilesB.append(testFileD)
self.testFilesB.append(testFileE)
uploadFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
datasetAction = uploadFactory(classname="NewDataset")
createAction = uploadFactory(classname="CreateBlocks")
datasetAction.execute(datasetPath=self.testDatasetA)
datasetAction.execute(datasetPath=self.testDatasetB)
self.blockAName = self.testDatasetA + "#" + makeUUID()
self.blockBName = self.testDatasetB + "#" + makeUUID()
newBlockA = DBSBlock(name=self.blockAName,
location="srm-cms.cern.ch",
das=None,
workflow=None)
newBlockA.setDataset(self.testDatasetA, 'data', 'VALID')
newBlockA.status = 'Closed'
newBlockB = DBSBlock(name=self.blockBName,
location="srm-cms.cern.ch",
das=None,
workflow=None)
newBlockB.setDataset(self.testDatasetB, 'data', 'VALID')
newBlockB.status = 'Closed'
createAction.execute(blocks=[newBlockA, newBlockB])
bufferFactory = DAOFactory(package="WMComponent.DBSBuffer.Database",
logger=myThread.logger,
dbinterface=myThread.dbi)
setBlock = bufferFactory(classname="DBSBufferFiles.SetBlock")
setBlock.execute(testFileA["lfn"], self.blockAName)
setBlock.execute(testFileB["lfn"], self.blockAName)
setBlock.execute(testFileC["lfn"], self.blockAName)
setBlock.execute(testFileD["lfn"], self.blockBName)
setBlock.execute(testFileE["lfn"], self.blockBName)
fileStatus = bufferFactory(classname="DBSBufferFiles.SetStatus")
fileStatus.execute(testFileA["lfn"], "LOCAL")
fileStatus.execute(testFileB["lfn"], "LOCAL")
fileStatus.execute(testFileC["lfn"], "LOCAL")
fileStatus.execute(testFileD["lfn"], "LOCAL")
fileStatus.execute(testFileE["lfn"], "LOCAL")
associateWorkflow = buffer3Factory(
classname="DBSBufferFiles.AssociateWorkflowToFile")
associateWorkflow.execute(testFileA["lfn"], "BogusRequest",
"BogusTask")
associateWorkflow.execute(testFileB["lfn"], "BogusRequest",
"BogusTask")
associateWorkflow.execute(testFileC["lfn"], "BogusRequest",
"BogusTask")
associateWorkflow.execute(testFileD["lfn"], "BogusRequest",
"BogusTask")
associateWorkflow.execute(testFileE["lfn"], "BogusRequest",
"BogusTask")
return
def createConfig(self):
"""
_createConfig_
Create a config for the PhEDExInjector with paths to the test DBS and
PhEDEx instances.
"""
config = self.testInit.getConfiguration()
config.component_("DBSInterface")
config.DBSInterface.globalDBSUrl = self.dbsURL
config.component_("PhEDExInjector")
config.PhEDExInjector.phedexurl = self.phedexURL
config.PhEDExInjector.subscribeMSS = True
config.PhEDExInjector.group = "Saturn"
config.PhEDExInjector.pollInterval = 30
config.PhEDExInjector.subscribeInterval = 60
return config
def retrieveReplicaInfoForBlock(self, blockName):
"""
_retrieveReplicaInfoForBlock_
Retrieve the replica information for a block. It takes several minutes
after a block is injected for the statistics to be calculated, so this
will block until that information is available.
"""
attempts = 0
while attempts < 15:
result = self.phedex.getReplicaInfoForFiles(block=blockName)
if "phedex" in result:
if "block" in result["phedex"]:
if len(result["phedex"]["block"]) != 0:
return result["phedex"]["block"][0]
attempts += 1
time.sleep(20)
logging.info("Could not retrieve replica info for block: %s" %
blockName)
return None
@attr("integration")
def testPoller(self):
"""
_testPoller_
Stuff the database and have the poller upload files to PhEDEx. Retrieve
replica information for the uploaded blocks and verify that all files
have been injected.
"""
return
self.stuffDatabase()
poller = PhEDExInjectorPoller(self.createConfig())
poller.setup(parameters=None)
poller.algorithm(parameters=None)
replicaInfo = self.retrieveReplicaInfoForBlock(self.blockAName)
goldenLFNs = []
for file in self.testFilesA:
goldenLFNs.append(file["lfn"])
for replicaFile in replicaInfo["file"]:
assert replicaFile["name"] in goldenLFNs, \
"Error: Extra file in replica block: %s" % replicaFile["name"]
goldenLFNs.remove(replicaFile["name"])
assert len(goldenLFNs) == 0, \
"Error: Files missing from PhEDEx replica: %s" % goldenLFNs
replicaInfo = self.retrieveReplicaInfoForBlock(self.blockBName)
goldenLFNs = []
for file in self.testFilesB:
goldenLFNs.append(file["lfn"])
for replicaFile in replicaInfo["file"]:
assert replicaFile["name"] in goldenLFNs, \
"Error: Extra file in replica block: %s" % replicaFile["name"]
goldenLFNs.remove(replicaFile["name"])
assert len(goldenLFNs) == 0, \
"Error: Files missing from PhEDEx replica: %s" % goldenLFNs
myThread = threading.currentThread()
daofactory = DAOFactory(package="WMComponent.DBSUpload.Database",
logger=myThread.logger,
dbinterface=myThread.dbi)
setBlock = daofactory(classname="SetBlockStatus")
setBlock.execute(self.blockAName,
locations=None,
open_status="InGlobalDBS")
poller.algorithm(parameters=None)
replicaInfo = self.retrieveReplicaInfoForBlock(self.blockAName)
assert replicaInfo["is_open"] == "n", \
"Error: block should be closed."
replicaInfo = self.retrieveReplicaInfoForBlock(self.blockBName)
assert replicaInfo["is_open"] == "y", \
"Error: block should be open."
return
def test_CustodialSiteA(self):
"""
_CustodialSiteA_
Check the custodialSite stuff by DAO, since I don't have a cert
First make sure we properly handle having no custodialSite
"""
self.stuffDatabase()
myThread = threading.currentThread()
daofactory = DAOFactory(package="WMComponent.PhEDExInjector.Database",
logger=myThread.logger,
dbinterface=myThread.dbi)
getUninjected = daofactory(classname="GetUninjectedFiles")
uninjectedFiles = getUninjected.execute()
self.assertEqual(uninjectedFiles.keys(), ['srm-cms.cern.ch'])
return
class FeederManagerTest(unittest.TestCase):
"""
TestCase for TestFeederManager module
"""
_maxMessage = 10
def setUp(self):
"""
_setUp_
Setup the database and logging connection. Try to create all needed
tables.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.generateWorkDir()
self.testInit.setSchema(customModules = \
['WMCore.Agent.Database',
'WMComponent.FeederManager.Database',
'WMCore.ThreadPool',
'WMCore.WMBS'],
useDefault = False)
return
def tearDown(self):
"""
_tearDown_
Database deletion
"""
self.testInit.clearDatabase()
return
def getConfig(self):
"""
_createConfig_
Create a config for the JobAccountant. This config needs to include
information for connecting to the database as the component will create
it's own database connections. These parameters are still pulled from
the environment.
"""
config = self.testInit.getConfiguration()
self.testInit.generateWorkDir(config)
config.component_("FeederManager")
config.FeederManager.logLevel = "INFO"
config.FeederManager.componentName = "FeederManager"
config.FeederManager.componentDir = \
os.path.join(os.getenv("TESTDIR"), "FeederManager")
config.FeederManager.addDatasetWatchHandler = \
'WMComponent.FeederManager.Handler.DefaultAddDatasetWatch'
# The maximum number of threads to process each message type
config.FeederManager.maxThreads = 10
# The poll interval at which to look for new fileset/feeder association
config.FeederManager.pollInterval = 60
return config
def testA(self):
"""
_testA_
Handle AddDatasetWatch events
"""
myThread = threading.currentThread()
config = self.getConfig()
testFeederManager = FeederManager(config)
testFeederManager.prepareToStart()
for i in xrange(0, FeederManagerTest._maxMessage):
for j in xrange(0, 3):
feederManagerdict = {'payload':{'FeederType':'NO Feeder',
'dataset' : 'NO DATASET', 'FileType' : 'NO FILE TYPE',
'StartRun' : 'NO START RUN' }}
testFeederManager.handleMessage( type = 'AddDatasetWatch',
payload = feederManagerdict )
time.sleep(30)
myThread.workerThreadManager.terminateWorkers()
while threading.activeCount() > 1:
print('Currently: '+str(threading.activeCount())+\
' Threads. Wait until all our threads have finished')
time.sleep(1)
class WMAgentTest(unittest.TestCase):
"""
_WMAgentTest_
Global unittest for all WMAgent components
"""
# This is an integration test
__integration__ = "Any old bollocks"
sites = ['T2_US_Florida', 'T2_US_UCSD', 'T2_TW_Taiwan', 'T1_CH_CERN']
components = ['JobCreator', 'JobSubmitter', 'JobTracker',
'JobAccountant', 'JobArchiver', 'TaskArchiver',
'RetryManager', 'ErrorHandler']
def setUp(self):
"""
_setUp_
Set up vital components
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMCore.WMBS",'WMCore.MsgService',
'WMCore.ResourceControl', 'WMCore.ThreadPool',
'WMCore.Agent.Database'],
useDefault = False)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = self.daoFactory(classname = "Locations.New")
pendingSlots = self.daoFactory(classname = "Locations.SetPendingSlots")
for site in self.sites:
locationAction.execute(siteName = site, pnn = 'se.%s' % (site), ceName = site)
pendingSlots.execute(siteName = site, pendingSlots = 1000)
#Create sites in resourceControl
resourceControl = ResourceControl()
for site in self.sites:
resourceControl.insertSite(siteName = site, pnn = 'se.%s' % (site), ceName = site)
resourceControl.insertThreshold(siteName = site, taskType = 'Processing', \
maxSlots = 10000, pendingSlots = 10000)
self.testDir = self.testInit.generateWorkDir()
# Set heartbeat
for component in self.components:
heartbeatAPI = HeartbeatAPI(component)
heartbeatAPI.registerComponent()
self.configFile = EmulatorSetup.setupWMAgentConfig()
return
def tearDown(self):
"""
_tearDown_
Tear down everything and go home.
"""
self.testInit.clearDatabase()
self.testInit.delWorkDir()
EmulatorSetup.deleteConfig(self.configFile)
return
def createTestWorkload(self, workloadName = 'Test', emulator = True):
"""
_createTestWorkload_
Creates a test workload for us to run on, hold the basic necessities.
"""
workload = testWorkload("TestWorkload")
rereco = workload.getTask("ReReco")
taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest'))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
workload.save(workloadName)
return workload
def getConfig(self):
"""
_getConfig_
This is the global test configuration object
"""
config = self.testInit.getConfiguration()
config.component_("Agent")
config.Agent.WMSpecDirectory = self.testDir
config.Agent.agentName = 'testAgent'
config.Agent.componentName = 'test'
# First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
# Now the CoreDatabase information
# This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# JobCreator
config.component_("JobCreator")
config.JobCreator.namespace = 'WMComponent.JobCreator.JobCreator'
config.JobCreator.logLevel = 'DEBUG'
config.JobCreator.maxThreads = 1
config.JobCreator.UpdateFromResourceControl = True
config.JobCreator.pollInterval = 10
config.JobCreator.jobCacheDir = self.testDir
config.JobCreator.defaultJobType = 'processing' #Type of jobs that we run, used for resource control
config.JobCreator.workerThreads = 2
config.JobCreator.componentDir = os.path.join(os.getcwd(), 'Components')
# JobSubmitter
config.component_("JobSubmitter")
config.JobSubmitter.namespace = 'WMComponent.JobSubmitter.JobSubmitter'
config.JobSubmitter.logLevel = 'INFO'
config.JobSubmitter.maxThreads = 1
config.JobSubmitter.pollInterval = 10
config.JobSubmitter.pluginName = 'CondorGlobusPlugin'
config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins'
config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit')
config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov')
config.JobSubmitter.submitScript = os.path.join(getWMBASE(),
'test/python/WMComponent_t/JobSubmitter_t',
'submit.sh')
config.JobSubmitter.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobSubmitter.workerThreads = 2
config.JobSubmitter.jobsPerWorker = 200
# JobTracker
config.component_("JobTracker")
config.JobTracker.logLevel = 'DEBUG'
config.JobTracker.pollInterval = 10
config.JobTracker.trackerName = 'CondorTracker'
config.JobTracker.pluginDir = 'WMComponent.JobTracker.Plugins'
config.JobTracker.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobTracker.runTimeLimit = 7776000 #Jobs expire after 90 days
config.JobTracker.idleTimeLimit = 7776000
config.JobTracker.heldTimeLimit = 7776000
config.JobTracker.unknTimeLimit = 7776000
# JobAccountant
config.component_("JobAccountant")
config.JobAccountant.pollInterval = 60
config.JobAccountant.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobAccountant.logLevel = 'INFO'
# JobArchiver
config.component_("JobArchiver")
config.JobArchiver.pollInterval = 60
config.JobArchiver.logLevel = 'INFO'
config.JobArchiver.logDir = os.path.join(self.testDir, 'logs')
config.JobArchiver.componentDir = os.path.join(os.getcwd(), 'Components')
config.JobArchiver.numberOfJobsToCluster = 1000
# Task Archiver
config.component_("TaskArchiver")
config.TaskArchiver.componentDir = self.testInit.generateWorkDir()
config.TaskArchiver.WorkQueueParams = {}
config.TaskArchiver.pollInterval = 60
config.TaskArchiver.logLevel = 'INFO'
config.TaskArchiver.timeOut = 0
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL',
'mnorman:[email protected]:5984')
config.JobStateMachine.couchDBName = "mnorman_test"
# Needed, because this is a test
os.makedirs(config.JobSubmitter.submitDir)
return config
def createFileCollection(self, name, nSubs, nFiles, workflowURL = 'test', site = None):
"""
_createFileCollection_
Create a collection of files for splitting into jobs
"""
myThread = threading.currentThread()
testWorkflow = Workflow(spec = workflowURL, owner = "mnorman",
name = name, task="/TestWorkload/ReReco")
testWorkflow.create()
for sub in range(nSubs):
nameStr = '%s-%i' % (name, sub)
testFileset = Fileset(name = nameStr)
testFileset.create()
for f in range(nFiles):
# pick a random site
if not site:
tmpSite = 'se.%s' % (random.choice(self.sites))
else:
tmpSite = 'se.%s' % (site)
testFile = File(lfn = "/lfn/%s/%i" % (nameStr, f), size = 1024, events = 10)
testFile.setLocation(tmpSite)
testFile.create()
testFileset.addFile(testFile)
testFileset.commit()
testFileset.markOpen(isOpen = 0)
testSubscription = Subscription(fileset = testFileset,
workflow = testWorkflow,
type = "Processing",
split_algo = "FileBased")
testSubscription.create()
return
def createReports(self, jobs, retryCount = 0):
"""
_createReports_
Create some dummy job reports for each job
"""
report = Report()
report.addStep('testStep', 0)
for job in jobs:
#reportPath = os.path.join(job['cache_dir'], 'Report.%i.pkl' % (retryCount))
reportPath = job['fwjr_path']
if os.path.exists(reportPath):
os.remove(reportPath)
report.save(reportPath)
return
def testA_StraightThrough(self):
"""
_StraightThrough_
Just run everything straight through without any variations
"""
# Do pre-submit job check
nRunning = getCondorRunningJobs()
self.assertEqual(nRunning, 0, "User currently has %i running jobs. Test will not continue" % (nRunning))
myThread = threading.currentThread()
workload = self.createTestWorkload()
config = self.getConfig()
name = 'WMAgent_Test1'
site = self.sites[0]
nSubs = 5
nFiles = 10
workloadPath = os.path.join(self.testDir, 'workloadTest',
'TestWorkload', 'WMSandbox',
'WMWorkload.pkl')
# Create a collection of files
self.createFileCollection(name = name, nSubs = nSubs,
nFiles = nFiles,
workflowURL = workloadPath,
site = site)
############################################################
# Test the JobCreator
config.Agent.componentName = 'JobCreator'
testJobCreator = JobCreatorPoller(config = config)
testJobCreator.algorithm()
time.sleep(5)
# Did all jobs get created?
getJobsAction = self.daoFactory(classname = "Jobs.GetAllJobs")
result = getJobsAction.execute(state = 'Created', jobType = "Processing")
self.assertEqual(len(result), nSubs*nFiles)
# Count database objects
result = myThread.dbi.processData('SELECT * FROM wmbs_sub_files_acquired')[0].fetchall()
self.assertEqual(len(result), nSubs * nFiles)
# Find the test directory
testDirectory = os.path.join(self.testDir, 'TestWorkload', 'ReReco')
self.assertTrue('JobCollection_1_0' in os.listdir(testDirectory))
self.assertTrue(len(os.listdir(testDirectory)) <= 20)
groupDirectory = os.path.join(testDirectory, 'JobCollection_1_0')
# First job should be in here
self.assertTrue('job_1' in os.listdir(groupDirectory))
jobFile = os.path.join(groupDirectory, 'job_1', 'job.pkl')
self.assertTrue(os.path.isfile(jobFile))
f = open(jobFile, 'r')
job = pickle.load(f)
f.close()
self.assertEqual(job['workflow'], name)
self.assertEqual(len(job['input_files']), 1)
self.assertEqual(os.path.basename(job['sandbox']), 'TestWorkload-Sandbox.tar.bz2')
###############################################################
# Now test the JobSubmitter
config.Agent.componentName = 'JobSubmitter'
testJobSubmitter = JobSubmitterPoller(config = config)
testJobSubmitter.algorithm()
# Check that jobs are in the right state
result = getJobsAction.execute(state = 'Created', jobType = "Processing")
self.assertEqual(len(result), 0)
result = getJobsAction.execute(state = 'Executing', jobType = "Processing")
self.assertEqual(len(result), nSubs * nFiles)
# Check assigned locations
getLocationAction = self.daoFactory(classname = "Jobs.GetLocation")
for id in result:
loc = getLocationAction.execute(jobid = id)
self.assertEqual(loc, [[site]])
# Check to make sure we have running jobs
nRunning = getCondorRunningJobs()
self.assertEqual(nRunning, nFiles * nSubs)
#################################################################
# Now the JobTracker
config.Agent.componentName = 'JobTracker'
testJobTracker = JobTrackerPoller(config = config)
testJobTracker.setup()
testJobTracker.algorithm()
# Running the algo without removing the jobs should do nothing
result = getJobsAction.execute(state = 'Executing', jobType = "Processing")
self.assertEqual(len(result), nSubs * nFiles)
condorRM()
time.sleep(1)
# All jobs gone?
nRunning = getCondorRunningJobs()
self.assertEqual(nRunning, 0)
testJobTracker.algorithm()
time.sleep(5)
# Running the algo without removing the jobs should do nothing
result = getJobsAction.execute(state = 'Executing', jobType = "Processing")
self.assertEqual(len(result), 0)
result = getJobsAction.execute(state = 'Complete', jobType = "Processing")
self.assertEqual(len(result), nSubs * nFiles)
#################################################################
# Now the JobAccountant
# First you need to load all jobs
self.getFWJRAction = self.daoFactory(classname = "Jobs.GetFWJRByState")
completeJobs = self.getFWJRAction.execute(state = "complete")
# Create reports for all jobs
self.createReports(jobs = completeJobs, retryCount = 0)
config.Agent.componentName = 'JobAccountant'
testJobAccountant = JobAccountantPoller(config = config)
testJobAccountant.setup()
# It should do something with the jobs
testJobAccountant.algorithm()
# All the jobs should be done now
result = getJobsAction.execute(state = 'Complete', jobType = "Processing")
self.assertEqual(len(result), 0)
result = getJobsAction.execute(state = 'Success', jobType = "Processing")
self.assertEqual(len(result), nSubs * nFiles)
#######################################################################
# Now the JobArchiver
config.Agent.componentName = 'JobArchiver'
testJobArchiver = JobArchiverPoller(config = config)
testJobArchiver.algorithm()
# All the jobs should be cleaned up
result = getJobsAction.execute(state = 'Success', jobType = "Processing")
self.assertEqual(len(result), 0)
result = getJobsAction.execute(state = 'Cleanout', jobType = "Processing")
self.assertEqual(len(result), nSubs * nFiles)
logDir = os.path.join(self.testDir, 'logs')
for job in completeJobs:
self.assertFalse(os.path.exists(job['fwjr_path']))
jobFolder = 'JobCluster_%i' \
% (int(job['id']/config.JobArchiver.numberOfJobsToCluster))
jobPath = os.path.join(logDir, jobFolder, 'Job_%i.tar' %(job['id']))
self.assertTrue(os.path.isfile(jobPath))
self.assertTrue(os.path.getsize(jobPath) > 0)
###########################################################################
# Now the TaskAchiver
config.Agent.componentName = 'TaskArchiver'
testTaskArchiver = TaskArchiverPoller(config = config)
testTaskArchiver.algorithm()
result = getJobsAction.execute(state = 'Cleanout', jobType = "Processing")
self.assertEqual(len(result), 0)
for jdict in completeJobs:
job = Job(id = jdict['id'])
self.assertFalse(job.exists())
if os.path.isdir('testDir'):
shutil.rmtree('testDir')
shutil.copytree('%s' %self.testDir, os.path.join(os.getcwd(), 'testDir'))
return
class ReportIntegrationTest(unittest.TestCase):
"""
_ReportIntegrationTest_
"""
def setUp(self):
"""
_setUp_
Setup the database and WMBS for the test.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer",
"WMCore.WMBS"],
useDefault = False)
myThread = threading.currentThread()
self.daofactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.dbsfactory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = self.daofactory(classname = "Locations.New")
locationAction.execute(siteName = "site1", seName = "cmssrm.fnal.gov")
inputFile = File(lfn = "/path/to/some/lfn", size = 10, events = 10,
locations = "cmssrm.fnal.gov")
inputFile.create()
inputFileset = Fileset(name = "InputFileset")
inputFileset.create()
inputFileset.addFile(inputFile)
inputFileset.commit()
unmergedFileset = Fileset(name = "UnmergedFileset")
unmergedFileset.create()
mergedFileset = Fileset(name = "MergedFileset")
mergedFileset.create()
procWorkflow = Workflow(spec = "wf001.xml", owner = "Steve",
name = "TestWF", task = "/TestWF/None")
procWorkflow.create()
procWorkflow.addOutput("outputRECORECO", unmergedFileset)
mergeWorkflow = Workflow(spec = "wf002.xml", owner = "Steve",
name = "MergeWF", task = "/MergeWF/None")
mergeWorkflow.create()
mergeWorkflow.addOutput("Merged", mergedFileset)
insertWorkflow = self.dbsfactory(classname = "InsertWorkflow")
insertWorkflow.execute("TestWF", "/TestWF/None", 0, 0, 0, 0)
insertWorkflow.execute("MergeWF", "/MergeWF/None", 0, 0, 0, 0)
self.procSubscription = Subscription(fileset = inputFileset,
workflow = procWorkflow,
split_algo = "FileBased",
type = "Processing")
self.procSubscription.create()
self.procSubscription.acquireFiles()
self.mergeSubscription = Subscription(fileset = unmergedFileset,
workflow = mergeWorkflow,
split_algo = "WMBSMergeBySize",
type = "Merge")
self.mergeSubscription.create()
self.procJobGroup = JobGroup(subscription = self.procSubscription)
self.procJobGroup.create()
self.mergeJobGroup = JobGroup(subscription = self.mergeSubscription)
self.mergeJobGroup.create()
self.testJob = Job(name = "testJob", files = [inputFile])
self.testJob.create(group = self.procJobGroup)
self.testJob["state"] = "complete"
myThread = threading.currentThread()
self.daofactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.stateChangeAction = self.daofactory(classname = "Jobs.ChangeState")
self.setFWJRAction = self.daofactory(classname = "Jobs.SetFWJRPath")
self.getJobTypeAction = self.daofactory(classname = "Jobs.GetType")
locationAction = self.daofactory(classname = "Locations.New")
locationAction.execute(siteName = "cmssrm.fnal.gov")
self.stateChangeAction.execute(jobs = [self.testJob])
self.tempDir = tempfile.mkdtemp()
return
def tearDown(self):
"""
_tearDown_
Clear out the database and the pickled report file.
"""
self.testInit.clearDatabase()
try:
os.remove(os.path.join(self.tempDir, "ProcReport.pkl"))
os.remove(os.path.join(self.tempDir, "MergeReport.pkl"))
except Exception as ex:
pass
try:
os.rmdir(self.tempDir)
except Exception as ex:
pass
return
def createConfig(self, workerThreads):
"""
_createConfig_
Create a config for the JobAccountant with the given number of worker
threads. This config needs to include information for connecting to the
database as the component will create it's own database connections.
These parameters are still pulled from the environment.
"""
config = self.testInit.getConfiguration()
self.testInit.generateWorkDir(config)
config.section_("JobStateMachine")
config.JobStateMachine.couchurl = os.getenv("COUCHURL")
config.JobStateMachine.couchDBName = "report_integration_t"
config.JobStateMachine.jobSummaryDBName = "report_integration_wmagent_summary_t"
config.component_("JobAccountant")
config.JobAccountant.pollInterval = 60
config.JobAccountant.workerThreads = workerThreads
config.JobAccountant.componentDir = os.getcwd()
config.JobAccountant.logLevel = 'SQLDEBUG'
config.component_("TaskArchiver")
config.TaskArchiver.localWMStatsURL = "%s/%s" % (config.JobStateMachine.couchurl, config.JobStateMachine.jobSummaryDBName)
return config
def verifyJobSuccess(self, jobID):
"""
_verifyJobSuccess_
Verify that the metadata for a successful job is correct. This will
check the outcome, retry count and state.
"""
testJob = Job(id = jobID)
testJob.load()
assert testJob["state"] == "success", \
"Error: test job in wrong state: %s" % testJob["state"]
assert testJob["retry_count"] == 0, \
"Error: test job has wrong retry count: %s" % testJob["retry_count"]
assert testJob["outcome"] == "success", \
"Error: test job has wrong outcome: %s" % testJob["outcome"]
return
def verifyFileMetaData(self, jobID, fwkJobReportFiles):
"""
_verifyFileMetaData_
Verify that all the files that were output by a job made it into WMBS
correctly. Compare the contents of WMBS to the files in the frameworks
job report.
Note that fwkJobReportFiles is a list of DataStructs File objects.
"""
testJob = Job(id = jobID)
testJob.loadData()
inputLFNs = []
for inputFile in testJob["input_files"]:
inputLFNs.append(inputFile["lfn"])
for fwkJobReportFile in fwkJobReportFiles:
outputFile = File(lfn = fwkJobReportFile["lfn"])
outputFile.loadData(parentage = 1)
assert outputFile["events"] == int(fwkJobReportFile["events"]), \
"Error: Output file has wrong events: %s, %s" % \
(outputFile["events"], fwkJobReportFile["events"])
assert outputFile["size"] == int(fwkJobReportFile["size"]), \
"Error: Output file has wrong size: %s, %s" % \
(outputFile["size"], fwkJobReportFile["size"])
for ckType in fwkJobReportFile["checksums"].keys():
assert ckType in outputFile["checksums"].keys(), \
"Error: Output file is missing checksums: %s" % ckType
assert outputFile["checksums"][ckType] == fwkJobReportFile["checksums"][ckType], \
"Error: Checksums don't match."
assert len(fwkJobReportFile["checksums"].keys()) == \
len(outputFile["checksums"].keys()), \
"Error: Wrong number of checksums."
jobType = self.getJobTypeAction.execute(jobID = jobID)
if jobType == "Merge":
assert str(outputFile["merged"]) == "True", \
"Error: Merge jobs should output merged files."
else:
assert outputFile["merged"] == fwkJobReportFile["merged"], \
"Error: Output file merged output is wrong: %s, %s" % \
(outputFile["merged"], fwkJobReportFile["merged"])
assert len(outputFile["locations"]) == 1, \
"Error: outputfile should have one location: %s" % outputFile["locations"]
assert list(outputFile["locations"])[0] == list(fwkJobReportFile["locations"])[0], \
"Error: wrong location for file."
assert len(outputFile["parents"]) == len(inputLFNs), \
"Error: Output file has wrong number of parents."
for outputParent in outputFile["parents"]:
assert outputParent["lfn"] in inputLFNs, \
"Error: Unknown parent file: %s" % outputParent["lfn"]
fwjrRuns = {}
for run in fwkJobReportFile["runs"]:
fwjrRuns[run.run] = run.lumis
for run in outputFile["runs"]:
assert run.run in fwjrRuns, \
"Error: Extra run in output: %s" % run.run
for lumi in run:
assert lumi in fwjrRuns[run.run], \
"Error: Extra lumi: %s" % lumi
fwjrRuns[run.run].remove(lumi)
if len(fwjrRuns[run.run]) == 0:
del fwjrRuns[run.run]
assert len(fwjrRuns.keys()) == 0, \
"Error: Missing runs, lumis: %s" % fwjrRuns
testJobGroup = JobGroup(id = testJob["jobgroup"])
testJobGroup.loadData()
jobGroupFileset = testJobGroup.output
jobGroupFileset.loadData()
assert outputFile["id"] in jobGroupFileset.getFiles(type = "id"), \
"Error: output file not in jobgroup fileset."
if testJob["mask"]["FirstEvent"] == None:
assert outputFile["first_event"] == 0, \
"Error: first event not set correctly: 0, %s" % \
outputFile["first_event"]
else:
assert testJob["mask"]["FirstEvent"] == outputFile["first_event"], \
"Error: last event not set correctly: %s, %s" % \
(testJob["mask"]["FirstEvent"], outputFile["first_event"])
return
def testReportHandling(self):
"""
_testReportHandling_
Verify that we're able to parse a CMSSW report, convert it to a Report()
style report, pickle it and then have the accountant process it.
"""
self.procPath = os.path.join(WMCore.WMBase.getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml")
myReport = Report("cmsRun1")
myReport.parse(self.procPath)
# Fake some metadata that should be added by the stageout scripts.
for fileRef in myReport.getAllFileRefsFromStep("cmsRun1"):
fileRef.size = 1024
fileRef.location = "cmssrm.fnal.gov"
fwjrPath = os.path.join(self.tempDir, "ProcReport.pkl")
cmsRunStep = myReport.retrieveStep("cmsRun1")
cmsRunStep.status = 0
myReport.setTaskName('/TestWF/None')
myReport.persist(fwjrPath)
self.setFWJRAction.execute(jobID = self.testJob["id"], fwjrPath = fwjrPath)
pFile = DBSBufferFile(lfn = "/path/to/some/lfn", size = 600000, events = 60000)
pFile.setAlgorithm(appName = "cmsRun", appVer = "UNKNOWN",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
pFile.setDatasetPath("/bogus/dataset/path")
#pFile.addRun(Run(1, *[45]))
pFile.create()
config = self.createConfig(workerThreads = 1)
accountant = JobAccountantPoller(config)
accountant.setup()
accountant.algorithm()
self.verifyJobSuccess(self.testJob["id"])
self.verifyFileMetaData(self.testJob["id"], myReport.getAllFilesFromStep("cmsRun1"))
inputFile = File(lfn = "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR09_R_34X_V5_All_v1/0000/outputRECORECO.root")
inputFile.load()
self.testMergeJob = Job(name = "testMergeJob", files = [inputFile])
self.testMergeJob.create(group = self.mergeJobGroup)
self.testMergeJob["state"] = "complete"
self.stateChangeAction.execute(jobs = [self.testMergeJob])
self.mergePath = os.path.join(WMCore.WMBase.getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWMergeReport.xml")
myReport = Report("mergeReco")
myReport.parse(self.mergePath)
# Fake some metadata that should be added by the stageout scripts.
for fileRef in myReport.getAllFileRefsFromStep("mergeReco"):
fileRef.size = 1024
fileRef.location = "cmssrm.fnal.gov"
fileRef.dataset = {"applicationName": "cmsRun", "applicationVersion": "CMSSW_3_4_2_patch1",
"primaryDataset": "MinimumBias", "processedDataset": "Rereco-v1",
"dataTier": "RECO"}
fwjrPath = os.path.join(self.tempDir, "MergeReport.pkl")
myReport.setTaskName('/MergeWF/None')
cmsRunStep = myReport.retrieveStep("mergeReco")
cmsRunStep.status = 0
myReport.persist(fwjrPath)
self.setFWJRAction.execute(jobID = self.testMergeJob["id"], fwjrPath = fwjrPath)
accountant.algorithm()
self.verifyJobSuccess(self.testMergeJob["id"])
self.verifyFileMetaData(self.testMergeJob["id"], myReport.getAllFilesFromStep("mergeReco"))
return
class ConfigTest(unittest.TestCase):
"""
Test class for the WMAgent configs
"""
def setUp(self):
"""
_setUp_
Create the whole database
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMCore.WMBS",
"WMComponent.DBSBuffer.Database",
'WMCore.ResourceControl',
'WMCore.BossAir'],
useDefault = False)
self.testDir = self.testInit.generateWorkDir()
return
def tearDown(self):
"""
_tearDown_
Tear things down and go home
"""
self.testInit.clearDatabase()
return
def initComponents(self, configPath):
"""
_initComponents_
Start up the various components using the config
"""
if os.path.isfile(configPath):
# Read the config
config = loadConfigurationFile(configPath)
else:
msg = "No config file at desired location"
logging.error(msg)
raise Exception(msg)
masterConfig = self.testInit.getConfiguration()
config.Agent.useHeartbeat = False
config.CoreDatabase.socket = masterConfig.CoreDatabase.socket
config.CoreDatabase.connectUrl = masterConfig.CoreDatabase.connectUrl
# Have to do this because the agent hard codes its dirs
oldWorkDir = config.General.workDir
for compName in (config.listComponents_() + config.listWebapps_()):
component = getattr(config, compName)
for var in component.listSections_():
value = getattr(component, var)
if type(value) == str:
if re.search(oldWorkDir, value):
# Then set it
setattr(component, var, value.replace(oldWorkDir, self.testDir))
elif type(value) == list:
# Go through it one component at a time
for element in list:
if type(element) == str and re.search(oldWorkDir, element):
index = value.index(element)
value.remove(element)
value.insert(index, value.replace(oldWorkDir, self.testDir))
setattr(component, var, value)
elif type(value) == dict:
for key in value.keys():
if type(value[key]) == str and re.search(oldWorkDir, value[key]):
value[key] = value[key].replace(oldWorkDir, self.testDir)
setattr(component, var, value)
compList = (config.listComponents_() + config.listWebapps_())
components = []
# Get all components
components.append(JobCreator(config = config))
components.append(JobSubmitter(config = config))
components.append(JobTracker(config = config))
components.append(JobAccountant(config = config))
components.append(JobArchiver(config = config))
components.append(TaskArchiver(config = config))
components.append(ErrorHandler(config = config))
components.append(RetryManager(config = config))
components.append(DBSUpload(config = config))
# Now the optional ones
if 'PhEDExInjector' in compList:
components.append(PhEDExInjector(config = config))
# Init threads:
for component in components:
component.initInThread()
# preInitialize
for component in components:
component.preInitialization()
for component in components:
component.prepareToStop()
return
def testA_WMAgentConfig(self):
"""
_WMAgentConfig_
Test the WMAgentConfig file in WMCORE/etc
"""
# Get the config
configPath = os.path.join(WMCore.WMInit.getWMBASE(),
'etc', 'WMAgentConfig.py')
self.initComponents(configPath = configPath)
return
@attr('integration')
def testB_WMAgentPromptSkimConfig(self):
"""
_WMAgentConfig_
Test the WMAgentConfig file in WMCORE/etc
"""
# Get the config
configPath = os.path.join(WMCore.WMInit.getWMBASE(),
'etc', 'WMAgentPromptSkimConfig.py')
self.initComponents(configPath = configPath)
return
class PhEDExInjectorSubscriberTest(unittest.TestCase):
"""
_PhEDExInjectorSubscriberTest_
Unit tests for the PhEDExInjectorSubscriber.
Create some database inside DBSBuffer, run the subscriber algorithm
using a PhEDEx emulator and verify that it works both in unsafe and safe mode.
For unsafe mode there a WMBS database is also created
"""
def setUp(self):
"""
_setUp_
Install the DBSBuffer schema into the database and connect to PhEDEx.
"""
self.phedexURL = "https://bogus.cern.ch/bogus"
self.dbsURL = "https://bogus.cern.ch/bogus"
EmulatorHelper.setEmulators(phedex = True, dbs = True, siteDB = True)
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer",
"WMCore.WMBS"],
useDefault = False)
self.testFilesA = []
self.testFilesB = []
self.testDatasetA = "/BogusPrimary/Run2012Z-PromptReco-v1/RECO"
self.testDatasetB = "/BogusPrimary/CRUZET11-v1/RAW"
return
def tearDown(self):
"""
_tearDown_
Delete the database.
"""
self.testInit.clearDatabase()
EmulatorHelper.resetEmulators()
def createConfig(self, safeMode):
"""
_createConfig_
Create a config for the PhEDExInjector, paths to DBS and PhEDEx are dummies because
we are using Emulators
"""
config = self.testInit.getConfiguration()
config.component_("DBSInterface")
config.DBSInterface.globalDBSUrl = self.dbsURL
config.component_("PhEDExInjector")
config.PhEDExInjector.phedexurl = self.phedexURL
config.PhEDExInjector.subscribeDatasets = True
config.PhEDExInjector.group = "Saturn"
config.PhEDExInjector.pollInterval = 30
config.PhEDExInjector.subscribeInterval = 60
config.PhEDExInjector.safeOperationMode = safeMode
return config
def stuffDatabase(self):
"""
_stuffDatabase_
Fill the dbsbuffer with some files and blocks. We'll insert a total
of 5 files spanning two blocks. There will be a total of two datasets
inserted into the database, both from the same workflow.
All files will be already in GLOBAL and in_phedex
"""
myThread = threading.currentThread()
buffer3Factory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
insertWorkflow = buffer3Factory(classname = "InsertWorkflow")
insertWorkflow.execute("BogusRequest", "BogusTask", os.path.join(getTestBase(),
"WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl"))
checksums = {"adler32": "1234", "cksum": "5678"}
testFileA = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileA.setDatasetPath(self.testDatasetA)
testFileA.addRun(Run(2, *[45]))
testFileA.create()
testFileB = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileB.setDatasetPath(self.testDatasetA)
testFileB.addRun(Run(2, *[45]))
testFileB.create()
testFileC = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileC.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileC.setDatasetPath(self.testDatasetA)
testFileC.addRun(Run(2, *[45]))
testFileC.create()
self.testFilesA.append(testFileA)
self.testFilesA.append(testFileB)
self.testFilesA.append(testFileC)
testFileD = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileD.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileD.setDatasetPath(self.testDatasetB)
testFileD.addRun(Run(2, *[45]))
testFileD.create()
testFileE = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileE.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileE.setDatasetPath(self.testDatasetB)
testFileE.addRun(Run(2, *[45]))
testFileE.create()
self.testFilesB.append(testFileD)
self.testFilesB.append(testFileE)
uploadFactory = DAOFactory(package = "WMComponent.DBSUpload.Database",
logger = myThread.logger,
dbinterface = myThread.dbi)
createBlock = uploadFactory(classname = "SetBlockStatus")
self.blockAName = self.testDatasetA + "#" + makeUUID()
self.blockBName = self.testDatasetB + "#" + makeUUID()
createBlock.execute(block = self.blockAName, locations = ["srm-cms.cern.ch"], open_status = 0)
createBlock.execute(block = self.blockBName, locations = ["srm-cms.cern.ch"], open_status = 0)
bufferFactory = DAOFactory(package = "WMComponent.DBSBuffer.Database",
logger = myThread.logger,
dbinterface = myThread.dbi)
setBlock = bufferFactory(classname = "DBSBufferFiles.SetBlock")
setBlock.execute(testFileA["lfn"], self.blockAName)
setBlock.execute(testFileB["lfn"], self.blockAName)
setBlock.execute(testFileC["lfn"], self.blockAName)
setBlock.execute(testFileD["lfn"], self.blockBName)
setBlock.execute(testFileE["lfn"], self.blockBName)
fileStatus = bufferFactory(classname = "DBSBufferFiles.SetStatus")
fileStatus.execute(testFileA["lfn"], "GLOBAL")
fileStatus.execute(testFileB["lfn"], "GLOBAL")
fileStatus.execute(testFileC["lfn"], "GLOBAL")
fileStatus.execute(testFileD["lfn"], "GLOBAL")
fileStatus.execute(testFileE["lfn"], "GLOBAL")
phedexStatus = bufferFactory(classname = "DBSBufferFiles.SetPhEDExStatus")
phedexStatus.execute(testFileA["lfn"], 1)
phedexStatus.execute(testFileB["lfn"], 1)
phedexStatus.execute(testFileC["lfn"], 1)
phedexStatus.execute(testFileD["lfn"], 1)
phedexStatus.execute(testFileE["lfn"], 1)
associateWorkflow = buffer3Factory(classname = "DBSBufferFiles.AssociateWorkflowToFile")
associateWorkflow.execute(testFileA["lfn"], "BogusRequest", "BogusTask")
associateWorkflow.execute(testFileB["lfn"], "BogusRequest", "BogusTask")
associateWorkflow.execute(testFileC["lfn"], "BogusRequest", "BogusTask")
associateWorkflow.execute(testFileD["lfn"], "BogusRequest", "BogusTask")
associateWorkflow.execute(testFileE["lfn"], "BogusRequest", "BogusTask")
return
def stuffWMBS(self):
"""
_stuffWMBS_
Inject the workflow in WMBS and add the subscriptions
"""
testWorkflow = Workflow(spec = os.path.join(getTestBase(),
"WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl"),
owner = "/CN=OU/DN=SomeoneWithPermissions",
name = "BogusRequest", task = "BogusTask", owner_vogroup = "", owner_vorole = "")
testWorkflow.create()
testMergeWorkflow = Workflow(spec = os.path.join(getTestBase(),
"WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl"),
owner = "/CN=OU/DN=SomeoneWithPermissions",
name = "BogusRequest", task = "BogusTask/Merge", owner_vogroup = "", owner_vorole = "")
testMergeWorkflow.create()
testWMBSFileset = Fileset(name = "TopFileset")
testWMBSFileset.create()
testWMBSFilesetUnmerged = Fileset(name = "UnmergedFileset")
testWMBSFilesetUnmerged.create()
testFileA = File(lfn = "/this/is/a/lfnA" , size = 1024, events = 10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10)
testFileB.addRun(Run(10, *[12314]))
testFileB.setLocation('malpaquet')
testFileA.create()
testFileB.create()
testWMBSFileset.addFile(testFileA)
testWMBSFilesetUnmerged.addFile(testFileB)
testWMBSFileset.commit()
testWMBSFilesetUnmerged.commit()
testSubscription = Subscription(fileset = testWMBSFileset,
workflow = testWorkflow)
testSubscription.create()
testSubscriptionMerge = Subscription(fileset = testWMBSFilesetUnmerged,
workflow = testMergeWorkflow,
type = "Merge")
testSubscriptionMerge.create()
return (testSubscription, testSubscriptionMerge)
def testUnsafeModeSubscriptions(self):
"""
_testUnsafeModeSubscriptions_
Tests that we can make custodial/non-custodial subscriptions on
unsafe operation mode, this time we don't need WMBS for anything.
All is subscribed in one go.
Check that the requests are correct.
"""
self.stuffDatabase()
config = self.createConfig(safeMode = False)
subscriber = PhEDExInjectorSubscriber(config)
subscriber.setup({})
subscriber.algorithm({})
phedexInstance = subscriber.phedex
subscriptions = phedexInstance.subRequests
# Let's check /BogusPrimary/Run2012Z-PromptReco-v1/RECO
# According to the spec, this should be custodial at T1_US_FNAL
# Non-custodial at T1_UK_RAL and T3_CO_Uniandes
# Autoapproved in all sites
# Priority is normal
self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed")
subInfoA = subscriptions[self.testDatasetA]
self.assertEqual(len(subInfoA), 3, "Dataset A was not subscribed to all sites")
for subInfo in subInfoA:
site = subInfo["node"][0]
self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription")
if site == "T1_UK_RAL_MSS" or site == "T3_CO_Uniandes":
self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"])
elif site == "T1_US_FNAL_MSS":
self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "y", "Wrong subscription type for dataset A at %s" % subInfo["node"])
else:
self.fail("Dataset A was subscribed to a wrong site %s" % site)
# Now check /BogusPrimary/CRUZET11-v1/RAW
# According to the spec, this is not custodial anywhere
# Non-custodial at T1_UK_RAL and T2_CH_CERN
# Request only at both sites and with high priority
self.assertTrue(self.testDatasetB in subscriptions, "Dataset B was not subscribed")
subInfoB = subscriptions[self.testDatasetB]
self.assertEqual(len(subInfoB), 2, "Dataset B was not subscribed to all sites")
for subInfo in subInfoB:
site = subInfo["node"][0]
self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription")
if site == "T1_UK_RAL_MSS" or site == "T2_CH_CERN":
self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset B at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset B at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset B at %s" % subInfo["node"])
else:
self.fail("Dataset B was subscribed to a wrong site %s" % site)
myThread = threading.currentThread()
result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset where subscribed = 1")[0].fetchall()
self.assertEqual(result[0][0], 2, "Not all datasets were marked as subscribed")
return
def testSafeModeSubscriptions(self):
"""
_testSafeModeSubscriptions_
Tests that we can make custodial/non-custodial subscriptions on
safe operation mode, make sure that the flow of subscriptions
obeys the rule laid in the subscriber documentation.
Check that the requests are correct.
"""
config = self.createConfig(safeMode = True)
self.stuffDatabase()
topSubscription, mergeSubscription = self.stuffWMBS()
# Start the subscriber
subscriber = PhEDExInjectorSubscriber(config)
subscriber.setup({})
# Run once, this means that all custodial and non-custodial subscriptions
# will be made but none will be Move
subscriber.algorithm({})
phedexInstance = subscriber.phedex
subscriptions = phedexInstance.subRequests
# Let's check /BogusPrimary/Run2012Z-PromptReco-v1/RECO
# According to the spec, this should be custodial at T1_US_FNAL
# Non-custodial at T1_UK_RAL and T3_CO_Uniandes
# Autoapproved in all sites
# Priority is normal
self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed")
subInfoA = subscriptions[self.testDatasetA]
self.assertEqual(len(subInfoA), 3, "Dataset A was not subscribed to all sites")
for subInfo in subInfoA:
site = subInfo["node"][0]
self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription")
if site == "T1_UK_RAL_MSS" or site == "T3_CO_Uniandes":
self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"])
elif site == "T1_US_FNAL_MSS":
self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"])
else:
self.fail("Dataset A was subscribed to a wrong site %s" % site)
# Now check /BogusPrimary/CRUZET11-v1/RAW
# According to the spec, this is not custodial anywhere
# Non-custodial at T1_UK_RAL and T2_CH_CERN
# Request only at both sites and with high priority
self.assertTrue(self.testDatasetB in subscriptions, "Dataset B was not subscribed")
subInfoB = subscriptions[self.testDatasetB]
self.assertEqual(len(subInfoB), 2, "Dataset B was not subscribed to all sites")
for subInfo in subInfoB:
site = subInfo["node"][0]
self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription")
if site == "T1_UK_RAL_MSS" or site == "T2_CH_CERN":
self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset B at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset B at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset B at %s" % subInfo["node"])
else:
self.fail("Dataset B was subscribed to a wrong site %s" % site)
myThread = threading.currentThread()
result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset where subscribed = 1")[0].fetchall()
self.assertEqual(result[0][0], 2, "Not all datasets were marked as partially subscribed")
# Now finish the Processing subscription and run the algorithm again
topSubscription.markFinished()
subscriber.algorithm({})
self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed")
subInfoA = subscriptions[self.testDatasetA]
self.assertEqual(len(subInfoA), 4, "Dataset A was not subscribed again to custodial site")
moveCount = 0
for subInfo in subInfoA:
site = subInfo["node"][0]
self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription")
if site == "T1_UK_RAL_MSS" or site == "T3_CO_Uniandes":
self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"])
elif site == "T1_US_FNAL_MSS":
self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"])
if subInfo["move"] == "y":
moveCount += 1
else:
self.fail("Dataset A was subscribed to a wrong site %s" % site)
self.assertEqual(moveCount, 1, "Move subscription was not made")
self.assertTrue(self.testDatasetB in subscriptions, "Dataset B was not subscribed")
subInfoB = subscriptions[self.testDatasetB]
self.assertEqual(len(subInfoB), 2, "Dataset B was susbcribed again")
result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset where subscribed = 2")[0].fetchall()
self.assertEqual(result[0][0], 2, "Not all datasets were marked as subscribed")
return
class ResultSetTest(unittest.TestCase):
def setUp(self):
"""
Set up for initializing the ResultSet test class
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
#self.testInit.setSchema(customModules = ["WMCore.WMBS"],
# useDefault = False)
self.mydialect = self.testInit.getConfiguration().CoreDatabase.dialect
self.myThread = threading.currentThread()
if self.mydialect.lower() == 'mysql':
create_sql = "create table test (bind1 varchar(20), bind2 varchar(20)) ENGINE=InnoDB "
elif self.mydialect.lower() == 'sqlite':
create_sql = "create table test (bind1 varchar(20), bind2 varchar(20))"
else:
create_sql = "create table test (bind1 varchar(20), bind2 varchar(20))"
#Create a table and insert several pieces
self.myThread.dbi.processData(create_sql)
return
def tearDown(self):
"""
Delete the ResultSet test class
"""
sql = 'drop table test'
self.myThread.dbi.processData(sqlstmt = sql, conn = self.myThread.dbi.connection())
return
def testFullResultSet(self):
"""
Function to test all the functionality of the ResultSet
"""
print "testFullResultSet"
#ResultSet is a very simple function. This function should
#create a ResultSet, fill it with data, and then read it
#out, all in one go.
testSet = ResultSet()
testDict = {'bind1': 'value1', 'bind2':'value2'}
binds = [ {'bind1':'value3a', 'bind2': 'value2a'},
{'bind1':'value3b', 'bind2': 'value2b'},
{'bind1':'value3c', 'bind2': 'value2c'}]
self.myThread.dbi.processData('insert into test (bind1, bind2) values (:bind1, :bind2)', testDict)
self.myThread.dbi.processData('insert into test (bind1, bind2) values (:bind1, :bind2)', binds)
testProxy = self.myThread.dbi.connection().execute('select * from test')
#import pdb
#pdb.set_trace()
testSet.add(testProxy)
self.assertEqual(testProxy.rowcount, 4)
self.assertEqual(testSet.rowcount, 4)
#Test to make sure fetchone and fetchall both work
self.assertEqual(str(testSet.fetchone()[0]), 'value1')
self.assertEqual(str(testSet.fetchall()[-1][1]), 'value2c')
def testRowCount(self):
testSet = ResultSet()
insertProxy = self.myThread.dbi.connection().execute("insert into test (bind1, bind2) values ('a', 'b')")
testSet.add(insertProxy)
self.assertEqual(testSet.rowcount, 1)
updateProxy = self.myThread.dbi.connection().execute("update test set bind1 = 'c'")
testSet.add(updateProxy)
self.assertEqual(testSet.rowcount, 2)
updateProxy = self.myThread.dbi.connection().execute("update test set bind1 = 'c' where bind1 = 'a'")
testSet.add(updateProxy)
self.assertEqual(testSet.rowcount, 2)
insertProxy = self.myThread.dbi.connection().execute("insert into test (bind1, bind2) values ('a', 'b')")
testSet.add(insertProxy)
self.assertEqual(testSet.rowcount, 3)
selectProxy = self.myThread.dbi.connection().execute("select * from test")
testSet.add(selectProxy)
self.assertEqual(testSet.rowcount, 5)
selectProxy = self.myThread.dbi.connection().execute("delete from test")
testSet.add(selectProxy)
self.assertEqual(testSet.rowcount, 7)
def test1000Binds(self):
testSet2 = ResultSet()
#Now insert and select a huge number of values
#This depends on the times variable
binds = [ ]
largeInsert = [ ]
times = 1000
#For now I don't want to get too dependent on processData() with many binds, since right now
#it doesn't work. That does make things awkward though.
for i in range(times):
binds = {'bind1': 'value1'+str(i), 'bind2':'value2'+str(i)}
self.myThread.dbi.processData("insert into test (bind1, bind2) values (:bind1, :bind2)", binds)
sql = "select bind1 from test"
testProxy = self.myThread.dbi.connection().execute(sql)
testSet2.add(testProxy)
self.assertEqual(len(testSet2.fetchall()), times)
if self.mydialect.lower() == 'sqlite':
self.assertEqual(str(testSet2.fetchall()[1][0]), 'value11')
self.assertEqual(testSet2.fetchone()[0], None)
elif self.mydialect.lower() == 'mysql':
self.assertEqual(str(testSet2.fetchall()[1][0]), 'value11')
self.assertEqual(testSet2.fetchone()[0], 'value10')
elif self.mydialect.lower() == 'oracle':
self.assertEqual(str(testSet2.fetchall()[1][0]), 'value11')
self.assertEqual(testSet2.fetchone()[0], 'value10')
return
class TestRecoveryCode(unittest.TestCase):
"""
Test for recoveryCode; disaster recovery system
"""
#This is an integration test
__integration__ = "So this guy walks into a bar..."
def setUp(self):
"""
Mimic remains of destroyed job
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.nJobs = 10
os.mkdir('test')
os.mkdir('test/Basic')
os.mkdir('test/Basic/Crap')
os.mkdir('test/Basic/Crap/JobCollection_1')
for i in range(self.nJobs):
os.mkdir('test/Basic/Crap/JobCollection_1/Job_%i' % (i))
self.logLevel = 'INFO'
#There must be a submit directory for jobSubmitter
if not os.path.isdir('submit'):
os.mkdir('submit')
#There must be a log directory for jobArchiver
if not os.path.isdir('logs'):
os.mkdir('logs')
def tearDown(self):
"""
Check to make sure that everything is good.
"""
if os.path.isdir('test'):
shutil.rmtree('test')
for file in os.listdir('logs'):
#You have to remove the staged out logs
os.remove(os.path.join('logs', file))
def getConfig(self):
"""
_getConfig_
For now, build a config file from the ground up.
Later, use this as a model for the JSM master config
"""
myThread = threading.currentThread()
self.testInit = TestInit()
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
config = self.testInit.getConfiguration()
self.tempDir = self.testInit.generateWorkDir( config = config )
#Now we go by component
#First the JobCreator
config.component_("JobCreator")
config.JobCreator.namespace = 'WMComponent.JobCreator.JobCreator'
config.JobCreator.logLevel = self.logLevel
config.JobCreator.maxThreads = 1
config.JobCreator.UpdateFromSiteDB = True
config.JobCreator.pollInterval = 10
config.JobCreator.jobCacheDir = os.path.join(os.getcwd(), 'test')
config.JobCreator.defaultJobType = 'processing' #Type of jobs that we run, used for resource control
config.JobCreator.workerThreads = 2
config.JobCreator.componentDir = os.path.join(os.getcwd(), 'Components/JobCreator')
#JobMaker
config.component_('JobMaker')
config.JobMaker.logLevel = self.logLevel
config.JobMaker.namespace = 'WMCore.WMSpec.Makers.JobMaker'
config.JobMaker.maxThreads = 1
config.JobMaker.makeJobsHandler = 'WMCore.WMSpec.Makers.Handlers.MakeJobs'
#JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL', 'mnorman:[email protected]:5984')
config.JobStateMachine.couchDBName = "mnorman_test"
#JobSubmitter
config.component_("JobSubmitter")
config.JobSubmitter.logLevel = self.logLevel
config.JobSubmitter.maxThreads = 1
config.JobSubmitter.pollInterval = 10
config.JobSubmitter.pluginName = 'ShadowPoolPlugin'
config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins'
config.JobSubmitter.submitDir = os.path.join(os.getcwd(), 'submit')
config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov')
config.JobSubmitter.submitScript = os.path.join(os.getcwd(), 'submit.sh')
config.JobSubmitter.componentDir = os.path.join(os.getcwd(), 'Components/JobSubmitter')
config.JobSubmitter.inputFile = os.path.join(os.getcwd(), 'FrameworkJobReport-4540.xml')
config.JobSubmitter.workerThreads = 1
config.JobSubmitter.jobsPerWorker = 100
#JobTracker
config.component_("JobTracker")
config.JobTracker.logLevel = self.logLevel
config.JobTracker.pollInterval = 10
config.JobTracker.trackerName = 'TestTracker'
config.JobTracker.pluginDir = 'WMComponent.JobTracker.Plugins'
config.JobTracker.runTimeLimit = 7776000 #Jobs expire after 90 days
config.JobTracker.idleTimeLimit = 7776000
config.JobTracker.heldTimeLimit = 7776000
config.JobTracker.unknTimeLimit = 7776000
#ErrorHandler
config.component_("ErrorHandler")
config.ErrorHandler.logLevel = self.logLevel
config.ErrorHandler.namespace = 'WMComponent.ErrorHandler.ErrorHandler'
config.ErrorHandler.maxThreads = 30
config.ErrorHandler.maxRetries = 10
config.ErrorHandler.pollInterval = 10
#RetryManager
config.component_("RetryManager")
config.RetryManager.logLevel = self.logLevel
config.RetryManager.namespace = 'WMComponent.RetryManager.RetryManager'
config.RetryManager.maxRetries = 10
config.RetryManager.pollInterval = 10
config.RetryManager.coolOffTime = {'create': 10, 'submit': 10, 'job': 10}
config.RetryManager.pluginPath = 'WMComponent.RetryManager.PlugIns'
config.RetryManager.pluginName = ''
config.RetryManager.WMCoreBase = WMCore.WMInit.getWMBASE()
#JobAccountant
config.component_("JobAccountant")
config.JobAccountant.logLevel = self.logLevel
#config.JobAccountant.logLevel = 'SQLDEBUG'
config.JobAccountant.pollInterval = 10
config.JobAccountant.workerThreads = 1
config.JobAccountant.componentDir = os.path.join(os.getcwd(), 'Components/JobAccountant')
#JobArchiver
config.component_("JobArchiver")
config.JobArchiver.pollInterval = 10
config.JobArchiver.logLevel = self.logLevel
#config.JobArchiver.logLevel = 'SQLDEBUG'
config.JobArchiver.logDir = os.path.join(os.getcwd(), 'logs')
return config
def submitJobs(self, nJobs):
"""
_submitJobs_
Submit some broken jdls to the local condor submitter
"""
submitFile = """
universe = globus
globusscheduler = thisisadummyname.fnal.gov/jobmanager-suck
should_transfer_executable = TRUE
transfer_output_files = FrameworkJobReport.xml
should_transfer_files = YES
when_to_transfer_output = ON_EXIT
log_xml = True
notification = NEVER
Output = condor.$(Cluster).$(Process).out
Error = condor.$(Cluster).$(Process).err
Log = condor.$(Cluster).$(Process).log
Executable = /home/mnorman/WMCORE/test/python/WMCore_t/Operations_t/submit.sh
initialdir = /home/mnorman/WMCORE/test/python/WMCore_t/Operations_t/test/Basic/Crap/JobCollection_1/Job_%i
+WMAgent_JobName = \"65bf3894-d873-11de-9e40-0030482c2dd0-1\"
+WMAgent_JobID = %i
Queue 1
"""
for i in range(10):
f = open('submit/submit_%i.jdl' %(i), 'w')
f.write(submitFile % (i, i))
f.close()
command = ["condor_submit", 'submit/submit_%i.jdl' %(i)]
pipe = Popen(command, stdout = PIPE, stderr = PIPE, shell = False)
pipe.wait()
@attr('integration')
def testPurge(self):
"""
_testPurge_
Test the purge function, which should remove all job objects
"""
config = self.getConfig()
self.submitJobs(self.nJobs)
self.assertEqual(len(os.listdir('submit')), self.nJobs, 'Only found %i submit files' %(len(os.listdir('submit'))))
self.assertEqual(len(os.listdir('logs')), 0, 'Please empty the logs directory')
#Check that ten jobs were actually submitted
jobCheckString = os.popen('condor_q %s' %os.getenv('USER')).readlines()[-1]
self.assertEqual(jobCheckString, '%i jobs; %i idle, 0 running, 0 held\n' % (self.nJobs, self.nJobs))
purgeJobs = PurgeJobs(config)
purgeJobs.run()
self.assertEqual(os.listdir('test'), [])
self.assertEqual(len(os.listdir('logs')), self.nJobs, \
'Found %i tarballs instead of %i in logOut directory' \
%(len(os.listdir('logs')), self.nJobs) )
self.assertEqual(os.listdir('submit'), [])
#Check that jobs were actually removed
jobCheckString = os.popen('condor_q %s' %os.getenv('USER')).readlines()[-1]
self.assertEqual(jobCheckString, '0 jobs; 0 idle, 0 running, 0 held\n' )
return
class FeederManagerTest(unittest.TestCase):
"""
TestCase for TestFeederManager module
"""
_maxMessage = 10
def setUp(self):
"""
_setUp_
Setup the database and logging connection. Try to create all needed
tables.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.generateWorkDir()
self.testInit.setSchema(customModules = \
['WMCore.Agent.Database',
'WMComponent.FeederManager.Database',
'WMCore.ThreadPool',
'WMCore.WMBS'],
useDefault = False)
return
def tearDown(self):
"""
_tearDown_
Database deletion
"""
self.testInit.clearDatabase()
return
def getConfig(self):
"""
_getConfig_
Get defaults FeederManager parameters
"""
return self.testInit.getConfiguration(
os.path.join(WMCore.WMInit.getWMBASE(), \
'src/python/WMComponent/FeederManager/DefaultConfig.py'))
def testA(self):
"""
_testA_
Handle AddDatasetWatch events
"""
myThread = threading.currentThread()
config = self.getConfig()
testFeederManager = FeederManager(config)
testFeederManager.prepareToStart()
for i in xrange(0, FeederManagerTest._maxMessage):
for j in xrange(0, 3):
feederManagerdict = {'payload':{'FeederType':'NO Feeder', \
'dataset' : 'NO DATASET', 'FileType' : 'NO FILE TYPE', \
'StartRun' : 'NO START RUN' }}
testFeederManager.handleMessage( type = 'AddDatasetWatch' \
, payload = feederManagerdict )
time.sleep(30)
myThread.workerThreadManager.terminateWorkers()
while threading.activeCount() > 1:
print('Currently: '+str(threading.activeCount())+\
' Threads. Wait until all our threads have finished')
time.sleep(1)
class PhEDExInjectorSubscriberTest(unittest.TestCase):
"""
_PhEDExInjectorSubscriberTest_
Unit tests for the PhEDExInjectorSubscriber.
Create some database inside DBSBuffer, run the subscriber algorithm
using a PhEDEx emulator and verify that it works both in unsafe and safe mode.
For unsafe mode there a WMBS database is also created
"""
def setUp(self):
"""
_setUp_
Install the DBSBuffer schema into the database and connect to PhEDEx.
"""
self.phedexURL = "https://bogus.cern.ch/bogus"
self.dbsURL = "https://bogus.cern.ch/bogus"
EmulatorHelper.setEmulators(phedex = True, dbs = True, siteDB = True)
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer",
"WMCore.WMBS"],
useDefault = False)
self.testFilesA = []
self.testFilesB = []
self.testDatasetA = "/BogusPrimary/Run2012Z-PromptReco-v1/RECO"
self.testDatasetB = "/BogusPrimary/CRUZET11-v1/RAW"
return
def tearDown(self):
"""
_tearDown_
Delete the database.
"""
self.testInit.clearDatabase()
EmulatorHelper.resetEmulators()
def createConfig(self, tier0Mode = False):
"""
_createConfig_
Create a config for the PhEDExInjector, paths to DBS and PhEDEx are dummies because
we are using Emulators
"""
config = self.testInit.getConfiguration()
config.component_("DBSInterface")
config.DBSInterface.globalDBSUrl = self.dbsURL
config.component_("PhEDExInjector")
config.PhEDExInjector.phedexurl = self.phedexURL
config.PhEDExInjector.subscribeDatasets = True
config.PhEDExInjector.group = "Saturn"
config.PhEDExInjector.pollInterval = 30
config.PhEDExInjector.subscribeInterval = 60
config.PhEDExInjector.tier0Mode = tier0Mode
return config
def stuffDatabase(self, tier0Mode = False):
"""
_stuffDatabase_
Fill the dbsbuffer with some files and blocks. We'll insert a total
of 5 files spanning two blocks. There will be a total of two datasets
inserted into the database.
All files will be already in GLOBAL and in_phedex
"""
myThread = threading.currentThread()
buffer3Factory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
insertWorkflow = buffer3Factory(classname = "InsertWorkflow")
insertWorkflow.execute("BogusRequestA", "BogusTask",
0, 0, 0, 0)
insertWorkflow.execute("BogusRequestB", "BogusTask",
0, 0, 0, 0)
checksums = {"adler32": "1234", "cksum": "5678"}
testFileA = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileA.setDatasetPath(self.testDatasetA)
testFileA.addRun(Run(2, *[45]))
testFileA.create()
testFileB = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileB.setDatasetPath(self.testDatasetA)
testFileB.addRun(Run(2, *[45]))
testFileB.create()
testFileC = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileC.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileC.setDatasetPath(self.testDatasetA)
testFileC.addRun(Run(2, *[45]))
testFileC.create()
self.testFilesA.append(testFileA)
self.testFilesA.append(testFileB)
self.testFilesA.append(testFileC)
testFileD = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileD.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileD.setDatasetPath(self.testDatasetB)
testFileD.addRun(Run(2, *[45]))
testFileD.create()
testFileE = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileE.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileE.setDatasetPath(self.testDatasetB)
testFileE.addRun(Run(2, *[45]))
testFileE.create()
self.testFilesB.append(testFileD)
self.testFilesB.append(testFileE)
uploadFactory = DAOFactory(package = "WMComponent.DBSUpload.Database",
logger = myThread.logger,
dbinterface = myThread.dbi)
createBlock = uploadFactory(classname = "SetBlockStatus")
self.blockAName = self.testDatasetA + "#" + makeUUID()
self.blockBName = self.testDatasetB + "#" + makeUUID()
createBlock.execute(block = self.blockAName, locations = ["srm-cms.cern.ch"], open_status = 'Closed')
createBlock.execute(block = self.blockBName, locations = ["srm-cms.cern.ch"], open_status = 'Closed')
bufferFactory = DAOFactory(package = "WMComponent.DBSBuffer.Database",
logger = myThread.logger,
dbinterface = myThread.dbi)
setBlock = bufferFactory(classname = "DBSBufferFiles.SetBlock")
setBlock.execute(testFileA["lfn"], self.blockAName)
setBlock.execute(testFileB["lfn"], self.blockAName)
setBlock.execute(testFileC["lfn"], self.blockAName)
setBlock.execute(testFileD["lfn"], self.blockBName)
setBlock.execute(testFileE["lfn"], self.blockBName)
fileStatus = bufferFactory(classname = "DBSBufferFiles.SetStatus")
fileStatus.execute(testFileA["lfn"], "GLOBAL")
fileStatus.execute(testFileB["lfn"], "GLOBAL")
fileStatus.execute(testFileC["lfn"], "GLOBAL")
fileStatus.execute(testFileD["lfn"], "GLOBAL")
fileStatus.execute(testFileE["lfn"], "GLOBAL")
phedexStatus = bufferFactory(classname = "DBSBufferFiles.SetPhEDExStatus")
phedexStatus.execute(testFileA["lfn"], 1)
phedexStatus.execute(testFileB["lfn"], 1)
phedexStatus.execute(testFileC["lfn"], 1)
phedexStatus.execute(testFileD["lfn"], 1)
phedexStatus.execute(testFileE["lfn"], 1)
associateWorkflow = buffer3Factory(classname = "DBSBufferFiles.AssociateWorkflowToFile")
associateWorkflow.execute(testFileA["lfn"], "BogusRequestA", "BogusTask")
associateWorkflow.execute(testFileB["lfn"], "BogusRequestA", "BogusTask")
associateWorkflow.execute(testFileC["lfn"], "BogusRequestA", "BogusTask")
associateWorkflow.execute(testFileD["lfn"], "BogusRequestB", "BogusTask")
associateWorkflow.execute(testFileE["lfn"], "BogusRequestB", "BogusTask")
# Make the desired subscriptions
insertSubAction = buffer3Factory(classname = "NewSubscription")
datasetA = DBSBufferDataset(path = self.testDatasetA)
datasetB = DBSBufferDataset(path = self.testDatasetB)
workload = WMWorkloadHelper()
workload.load(os.path.join(getTestBase(), 'WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl'))
if tier0Mode:
# Override the settings
workload.setSubscriptionInformation(custodialSites = ["T0_CH_CERN", "T1_US_FNAL"],
nonCustodialSites = ["T3_CO_Uniandes"],
priority = "Normal", custodialSubType = "Replica",
autoApproveSites = ["T0_CH_CERN"],
dataTier = "RECO")
workload.setSubscriptionInformation(custodialSites = ["T0_CH_CERN", "T1_UK_RAL"],
nonCustodialSites = [],
autoApproveSites = [],
priority = "High", custodialSubType = "Replica",
dataTier = "RAW")
insertSubAction.execute(datasetA.exists(), workload.getSubscriptionInformation()[self.testDatasetA])
insertSubAction.execute(datasetB.exists(), workload.getSubscriptionInformation()[self.testDatasetB])
return
def stuffWMBS(self):
"""
_stuffWMBS_
Inject a workflow in WMBS and add the subscriptions
"""
testWorkflow = Workflow(spec = "bogus.xml",
owner = "/CN=OU/DN=SomeoneWithPermissions",
name = "BogusRequestB", task = "BogusTask", owner_vogroup = "", owner_vorole = "")
testWorkflow.create()
testMergeWorkflow = Workflow(spec = "bogus.xml",
owner = "/CN=OU/DN=SomeoneWithPermissions",
name = "BogusRequestB", task = "BogusTask/Merge", owner_vogroup = "", owner_vorole = "")
testMergeWorkflow.create()
testWMBSFileset = Fileset(name = "TopFileset")
testWMBSFileset.create()
testWMBSFilesetUnmerged = Fileset(name = "UnmergedFileset")
testWMBSFilesetUnmerged.create()
testFileA = File(lfn = "/this/is/a/lfnA" , size = 1024, events = 10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10)
testFileB.addRun(Run(10, *[12314]))
testFileB.setLocation('malpaquet')
testFileA.create()
testFileB.create()
testWMBSFileset.addFile(testFileA)
testWMBSFilesetUnmerged.addFile(testFileB)
testWMBSFileset.commit()
testWMBSFilesetUnmerged.commit()
testSubscription = Subscription(fileset = testWMBSFileset,
workflow = testWorkflow)
testSubscription.create()
testSubscriptionMerge = Subscription(fileset = testWMBSFilesetUnmerged,
workflow = testMergeWorkflow,
type = "Merge")
testSubscriptionMerge.create()
return (testSubscription, testSubscriptionMerge)
def testNormalModeSubscriptions(self):
"""
_testNormalModeSubscriptions_
Tests that we can make custodial/non-custodial subscriptions on
normal operation mode, this time we don't need WMBS for anything.
All is subscribed in one go.
Check that the requests are correct.
"""
self.stuffDatabase()
config = self.createConfig()
subscriber = PhEDExInjectorSubscriber(config)
subscriber.setup({})
subscriber.algorithm({})
phedexInstance = subscriber.phedex
subscriptions = phedexInstance.subRequests
# Let's check /BogusPrimary/Run2012Z-PromptReco-v1/RECO
# According to the spec, this should be custodial at T1_US_FNAL
# Non-custodial at T1_UK_RAL and T3_CO_Uniandes
# Autoapproved in all sites
# Priority is normal
self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed")
subInfoA = subscriptions[self.testDatasetA]
self.assertEqual(len(subInfoA), 3, "Dataset A was not subscribed to all sites")
for subInfo in subInfoA:
site = subInfo["node"]
self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription")
if site == "T1_UK_RAL_MSS" or site == "T3_CO_Uniandes":
self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"])
elif site == "T1_US_FNAL_MSS":
self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "y", "Wrong subscription type for dataset A at %s" % subInfo["node"])
else:
self.fail("Dataset A was subscribed to a wrong site %s" % site)
# Now check /BogusPrimary/CRUZET11-v1/RAW
# According to the spec, this is not custodial anywhere
# Non-custodial at T1_UK_RAL and T2_CH_CERN
# Request only at both sites and with high priority
self.assertTrue(self.testDatasetB in subscriptions, "Dataset B was not subscribed")
subInfoB = subscriptions[self.testDatasetB]
self.assertEqual(len(subInfoB), 2, "Dataset B was not subscribed to all sites")
for subInfo in subInfoB:
site = subInfo["node"]
self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription")
if site == "T1_UK_RAL_MSS" or site == "T2_CH_CERN":
self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset B at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset B at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset B at %s" % subInfo["node"])
else:
self.fail("Dataset B was subscribed to a wrong site %s" % site)
myThread = threading.currentThread()
result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset_subscription where subscribed = 1")[0].fetchall()
self.assertEqual(result[0][0], 5, "Not all datasets were marked as subscribed")
result = myThread.dbi.processData("SELECT site FROM dbsbuffer_dataset_subscription where subscribed = 0")[0].fetchall()
self.assertEqual(result[0][0], "T1_IT_CNAF", "A non-valid CMS site was subscribed")
# Reset and run again and make sure that no duplicate subscriptions are created
myThread.dbi.processData("UPDATE dbsbuffer_dataset_subscription SET subscribed = 0")
subscriber.algorithm({})
self.assertEqual(len(subscriptions[self.testDatasetA]), 3)
self.assertEqual(len(subscriptions[self.testDatasetB]), 2)
return
def testTier0ModeSubscriptions(self):
"""
_testTier0ModeSubscriptions_
Tests that we can make custodial/non-custodial subscriptions on
tier0 operation mode, custodial moves are made on block level.
"""
self.stuffDatabase(tier0Mode = True)
self.stuffWMBS()
config = self.createConfig(tier0Mode = True)
subscriber = PhEDExInjectorSubscriber(config)
subscriber.setup({})
subscriber.algorithm({})
phedexInstance = subscriber.phedex
subscriptions = phedexInstance.subRequests
# Let's check /BogusPrimary/Run2012Z-PromptReco-v1/RECO
# According to the spec, this should be custodial at T0_CH_CERN and T1_US_FNAL
# Non-custodial at T2_US_Vanderbilt
# Autoapproved at CERN
# Priority is normal
self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed")
subInfoA = subscriptions[self.testDatasetA]
self.assertEqual(len(subInfoA), 4, "Dataset A was not subscribed to all sites")
for subInfo in subInfoA:
site = subInfo["node"]
if subInfo['level'] == 'block':
self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "y", "Wrong subscription type for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["node"], "T0_CH_CERN_MSS", "Wrong node for dataset A, block subscription.")
self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription")
elif site == "T0_CH_CERN_MSS":
self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["level"], "dataset", "Wrong level for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription")
elif site == "T1_US_FNAL_MSS":
self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["level"], "dataset", "Wrong level for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription")
elif site == "T3_CO_Uniandes":
self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["level"], "dataset", "Wrong level for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription")
else:
self.fail("Dataset A was subscribed to a wrong site %s" % site)
self.assertEqual(len([x for x in subInfoA if x['level'] == 'block']), 1)
# Now check /BogusPrimary/CRUZET11-v1/RAW
# According to the spec, custodial to T0_CH_CERN_MSS and T1_UK_RAL_MSS
# Request only at both sites and with high priority
self.assertTrue(self.testDatasetB in subscriptions)
subInfoB = subscriptions[self.testDatasetB]
self.assertEqual(len(subInfoB), 2, "Dataset B was not subscribed to all sites")
for subInfo in subInfoB:
site = subInfo["node"]
if site == "T0_CH_CERN_MSS" or site == "T1_UK_RAL_MSS":
self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset B at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset B at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset B at %s" % subInfo["node"])
self.assertEqual(subInfo["level"], "dataset", "Wrong level for dataset B at %s" % subInfo["node"])
self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription")
else:
self.fail("Dataset B was subscribed to a wrong site %s" % site)
myThread = threading.currentThread()
result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset_subscription where subscribed = 1")[0].fetchall()
self.assertEqual(result[0][0], 5, "Custodial move datasets were marked as subscribed")
# Delete the wmbs entries
myThread.dbi.processData("DELETE FROM wmbs_workflow")
subscriber.algorithm({})
self.assertEqual(len(subscriptions[self.testDatasetA]), 5)
self.assertTrue(self.testDatasetB in subscriptions)
self.assertEqual(len(subscriptions[self.testDatasetB]), 3)
self.assertEqual(len([x for x in subscriptions[self.testDatasetB] if x['level'] == 'block']), 1)
return
class HarnessTest(unittest.TestCase):
"""
TestCase for TestComponent module
"""
tempDir = None
def setUp(self):
"""
setup for test.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema()
def tearDown(self):
"""
Delete database
"""
self.testInit.clearDatabase()
def testB(self):
raise nose.SkipTest
config = self.testInit.getConfiguration()
self.tempDir = self.testInit.generateWorkDir(config)
config.component_("TestComponent")
config.TestComponent.logLevel = 'INFO'
config.section_("General")
config.TestComponent.componentDir = os.path.join( \
self.tempDir, "Components/TestComponent1")
config.General.workDir = config.TestComponent.componentDir
os.makedirs( config.TestComponent.componentDir )
# as this is a test we build the string from our global environment
# parameters normally you put this straight into the DefaultConfig.py file:
# testInit.getConfiguration returns from the environment variable by default
testComponent = TestComponent(config)
testComponent.prepareToStart()
testComponent.handleMessage('LogState','')
testComponent.handleMessage('TestMessage1','TestMessag1Payload')
testComponent.handleMessage('TestMessage2','TestMessag2Payload')
testComponent.handleMessage('TestMessage3','TestMessag3Payload')
testComponent.handleMessage('TestMessage4','TestMessag4Payload')
testComponent.handleMessage('Logging.DEBUG','')
testComponent.handleMessage('Logging.WARNING','')
testComponent.handleMessage('Logging.CRITICAL','')
testComponent.handleMessage('Logging.ERROR','')
testComponent.handleMessage('Logging.INFO','')
testComponent.handleMessage('Logging.SQLDEBUG','')
testComponent.handleMessage('TestComponent:Logging.DEBUG','')
testComponent.handleMessage('TestComponent:Logging.WARNING','')
testComponent.handleMessage('TestComponent:Logging.CRITICAL','')
testComponent.handleMessage('TestComponent:Logging.ERROR','')
testComponent.handleMessage('TestComponent:Logging.INFO','')
testComponent.handleMessage('TestComponent:Logging.SQLDEBUG','')
# test a non existing message (to generate an error)
errorMsg = ''
try:
testComponent.handleMessage('NonExistingMessageType','')
except Exception,ex:
errorMsg = str(ex)
self.assertTrue(errorMsg.startswith('Message NonExistingMessageType with payload'))
class PhEDExInjectorSubscriberTest(unittest.TestCase):
"""
_PhEDExInjectorSubscriberTest_
Unit tests for the PhEDExInjectorSubscriber.
Create some database inside DBSBuffer, run the subscriber algorithm
using a PhEDEx emulator and verify that it works both in unsafe and safe mode.
For unsafe mode there a WMBS database is also created
"""
def setUp(self):
"""
_setUp_
Install the DBSBuffer schema into the database and connect to PhEDEx.
"""
self.phedexURL = "https://bogus.cern.ch/bogus"
self.dbsURL = "https://bogus.cern.ch/bogus"
EmulatorHelper.setEmulators(phedex = True, dbs = True, siteDB = True)
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer",
"WMCore.WMBS"],
useDefault = False)
self.testFilesA = []
self.testFilesB = []
self.testDatasetA = "/BogusPrimary/Run2012Z-PromptReco-v1/RECO"
self.testDatasetB = "/BogusPrimary/CRUZET11-v1/RAW"
return
def tearDown(self):
"""
_tearDown_
Delete the database.
"""
self.testInit.clearDatabase()
EmulatorHelper.resetEmulators()
def createConfig(self, tier0Mode = False):
"""
_createConfig_
Create a config for the PhEDExInjector, paths to DBS and PhEDEx are dummies because
we are using Emulators
"""
config = self.testInit.getConfiguration()
config.component_("DBSInterface")
config.DBSInterface.globalDBSUrl = self.dbsURL
config.component_("PhEDExInjector")
config.PhEDExInjector.phedexurl = self.phedexURL
config.PhEDExInjector.subscribeDatasets = True
config.PhEDExInjector.group = "Saturn"
config.PhEDExInjector.pollInterval = 30
config.PhEDExInjector.subscribeInterval = 60
config.PhEDExInjector.tier0Mode = tier0Mode
return config
def stuffDatabase(self, tier0Mode = False):
"""
_stuffDatabase_
Fill the dbsbuffer with some files and blocks. We'll insert a total
of 5 files spanning two blocks. There will be a total of two datasets
inserted into the database.
All files will be already in GLOBAL and in_phedex
"""
myThread = threading.currentThread()
buffer3Factory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
insertWorkflow = buffer3Factory(classname = "InsertWorkflow")
insertWorkflow.execute("BogusRequestA", "BogusTask",
0, 0, 0, 0)
insertWorkflow.execute("BogusRequestB", "BogusTask",
0, 0, 0, 0)
checksums = {"adler32": "1234", "cksum": "5678"}
testFileA = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileA.setDatasetPath(self.testDatasetA)
testFileA.addRun(Run(2, *[45]))
testFileA.create()
testFileB = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileB.setDatasetPath(self.testDatasetA)
testFileB.addRun(Run(2, *[45]))
testFileB.create()
testFileC = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileC.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileC.setDatasetPath(self.testDatasetA)
testFileC.addRun(Run(2, *[45]))
testFileC.create()
self.testFilesA.append(testFileA)
self.testFilesA.append(testFileB)
self.testFilesA.append(testFileC)
testFileD = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileD.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileD.setDatasetPath(self.testDatasetB)
testFileD.addRun(Run(2, *[45]))
testFileD.create()
testFileE = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileE.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileE.setDatasetPath(self.testDatasetB)
testFileE.addRun(Run(2, *[45]))
testFileE.create()
self.testFilesB.append(testFileD)
self.testFilesB.append(testFileE)
uploadFactory = DAOFactory(package = "WMComponent.DBSUpload.Database",
logger = myThread.logger,
dbinterface = myThread.dbi)
createBlock = uploadFactory(classname = "SetBlockStatus")
self.blockAName = self.testDatasetA + "#" + makeUUID()
self.blockBName = self.testDatasetB + "#" + makeUUID()
createBlock.execute(block = self.blockAName, locations = ["srm-cms.cern.ch"], open_status = 'Closed')
createBlock.execute(block = self.blockBName, locations = ["srm-cms.cern.ch"], open_status = 'Closed')
bufferFactory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
setBlock = bufferFactory(classname = "DBSBufferFiles.SetBlock")
setBlock.execute(testFileA["lfn"], self.blockAName)
setBlock.execute(testFileB["lfn"], self.blockAName)
setBlock.execute(testFileC["lfn"], self.blockAName)
setBlock.execute(testFileD["lfn"], self.blockBName)
setBlock.execute(testFileE["lfn"], self.blockBName)
fileStatus = bufferFactory(classname = "DBSBufferFiles.SetStatus")
fileStatus.execute(testFileA["lfn"], "GLOBAL")
fileStatus.execute(testFileB["lfn"], "GLOBAL")
fileStatus.execute(testFileC["lfn"], "GLOBAL")
fileStatus.execute(testFileD["lfn"], "GLOBAL")
fileStatus.execute(testFileE["lfn"], "GLOBAL")
phedexStatus = bufferFactory(classname = "DBSBufferFiles.SetPhEDExStatus")
phedexStatus.execute(testFileA["lfn"], 1)
phedexStatus.execute(testFileB["lfn"], 1)
phedexStatus.execute(testFileC["lfn"], 1)
phedexStatus.execute(testFileD["lfn"], 1)
phedexStatus.execute(testFileE["lfn"], 1)
associateWorkflow = buffer3Factory(classname = "DBSBufferFiles.AssociateWorkflowToFile")
associateWorkflow.execute(testFileA["lfn"], "BogusRequestA", "BogusTask")
associateWorkflow.execute(testFileB["lfn"], "BogusRequestA", "BogusTask")
associateWorkflow.execute(testFileC["lfn"], "BogusRequestA", "BogusTask")
associateWorkflow.execute(testFileD["lfn"], "BogusRequestB", "BogusTask")
associateWorkflow.execute(testFileE["lfn"], "BogusRequestB", "BogusTask")
# Make the desired subscriptions
insertSubAction = buffer3Factory(classname = "NewSubscription")
datasetA = DBSBufferDataset(path = self.testDatasetA)
datasetB = DBSBufferDataset(path = self.testDatasetB)
workload = WMWorkloadHelper()
workload.load(os.path.join(getTestBase(), 'WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl'))
if tier0Mode:
# Override the settings
workload.setSubscriptionInformation(custodialSites = ["T0_CH_CERN", "T1_US_FNAL"],
nonCustodialSites = ["T3_CO_Uniandes"],
priority = "Normal", custodialSubType = "Replica",
autoApproveSites = ["T0_CH_CERN"],
dataTier = "RECO")
workload.setSubscriptionInformation(custodialSites = ["T0_CH_CERN", "T1_UK_RAL"],
nonCustodialSites = [],
autoApproveSites = [],
priority = "High", custodialSubType = "Replica",
dataTier = "RAW")
insertSubAction.execute(datasetA.exists(), workload.getSubscriptionInformation()[self.testDatasetA])
insertSubAction.execute(datasetB.exists(), workload.getSubscriptionInformation()[self.testDatasetB])
return
def stuffWMBS(self):
"""
_stuffWMBS_
Inject a workflow in WMBS and add the subscriptions
"""
testWorkflow = Workflow(spec = "bogus.xml",
owner = "/CN=OU/DN=SomeoneWithPermissions",
name = "BogusRequestB", task = "BogusTask", owner_vogroup = "", owner_vorole = "")
testWorkflow.create()
testMergeWorkflow = Workflow(spec = "bogus.xml",
owner = "/CN=OU/DN=SomeoneWithPermissions",
name = "BogusRequestB", task = "BogusTask/Merge", owner_vogroup = "", owner_vorole = "")
testMergeWorkflow.create()
testWMBSFileset = Fileset(name = "TopFileset")
testWMBSFileset.create()
testWMBSFilesetUnmerged = Fileset(name = "UnmergedFileset")
testWMBSFilesetUnmerged.create()
testFileA = File(lfn = "/this/is/a/lfnA" , size = 1024, events = 10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10)
testFileB.addRun(Run(10, *[12314]))
testFileB.setLocation('malpaquet')
testFileA.create()
testFileB.create()
testWMBSFileset.addFile(testFileA)
testWMBSFilesetUnmerged.addFile(testFileB)
testWMBSFileset.commit()
testWMBSFilesetUnmerged.commit()
testSubscription = Subscription(fileset = testWMBSFileset,
workflow = testWorkflow)
testSubscription.create()
testSubscriptionMerge = Subscription(fileset = testWMBSFilesetUnmerged,
workflow = testMergeWorkflow,
type = "Merge")
testSubscriptionMerge.create()
return (testSubscription, testSubscriptionMerge)
def testNormalModeSubscriptions(self):
"""
_testNormalModeSubscriptions_
Tests that we can make custodial/non-custodial subscriptions on
normal operation mode, this time we don't need WMBS for anything.
All is subscribed in one go.
Check that the requests are correct.
"""
self.stuffDatabase()
config = self.createConfig()
subscriber = PhEDExInjectorSubscriber(config)
subscriber.setup({})
subscriber.algorithm({})
phedexInstance = subscriber.phedex
subscriptions = phedexInstance.subRequests
# Let's check /BogusPrimary/Run2012Z-PromptReco-v1/RECO
# According to the spec, this should be custodial at T1_US_FNAL
# Non-custodial at T1_UK_RAL and T3_CO_Uniandes
# Autoapproved in all sites
# Priority is normal
self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed")
subInfoA = subscriptions[self.testDatasetA]
self.assertEqual(len(subInfoA), 3, "Dataset A was not subscribed to all sites")
for subInfo in subInfoA:
site = subInfo["node"]
self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription")
if site == "T1_UK_RAL_MSS" or site == "T3_CO_Uniandes":
self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"])
elif site == "T1_US_FNAL_MSS":
self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "y", "Wrong subscription type for dataset A at %s" % subInfo["node"])
else:
self.fail("Dataset A was subscribed to a wrong site %s" % site)
# Now check /BogusPrimary/CRUZET11-v1/RAW
# According to the spec, this is not custodial anywhere
# Non-custodial at T1_UK_RAL and T2_CH_CERN
# Request only at both sites and with high priority
self.assertTrue(self.testDatasetB in subscriptions, "Dataset B was not subscribed")
subInfoB = subscriptions[self.testDatasetB]
self.assertEqual(len(subInfoB), 2, "Dataset B was not subscribed to all sites")
for subInfo in subInfoB:
site = subInfo["node"]
self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription")
if site == "T1_UK_RAL_MSS" or site == "T2_CH_CERN":
self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset B at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset B at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset B at %s" % subInfo["node"])
else:
self.fail("Dataset B was subscribed to a wrong site %s" % site)
myThread = threading.currentThread()
result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset_subscription where subscribed = 1")[0].fetchall()
self.assertEqual(result[0][0], 5, "Not all datasets were marked as subscribed")
result = myThread.dbi.processData("SELECT site FROM dbsbuffer_dataset_subscription where subscribed = 0")[0].fetchall()
self.assertEqual(result[0][0], "T1_IT_CNAF", "A non-valid CMS site was subscribed")
# Reset and run again and make sure that no duplicate subscriptions are created
myThread.dbi.processData("UPDATE dbsbuffer_dataset_subscription SET subscribed = 0")
subscriber.algorithm({})
self.assertEqual(len(subscriptions[self.testDatasetA]), 3)
self.assertEqual(len(subscriptions[self.testDatasetB]), 2)
return
def testTier0ModeSubscriptions(self):
"""
_testTier0ModeSubscriptions_
Tests that we can make custodial/non-custodial subscriptions on
tier0 operation mode, custodial moves are made on block level.
"""
self.stuffDatabase(tier0Mode = True)
self.stuffWMBS()
config = self.createConfig(tier0Mode = True)
subscriber = PhEDExInjectorSubscriber(config)
subscriber.setup({})
subscriber.algorithm({})
phedexInstance = subscriber.phedex
subscriptions = phedexInstance.subRequests
# Let's check /BogusPrimary/Run2012Z-PromptReco-v1/RECO
# According to the spec, this should be custodial at T0_CH_CERN and T1_US_FNAL
# Non-custodial at T2_US_Vanderbilt
# Autoapproved at CERN
# Priority is normal
self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed")
subInfoA = subscriptions[self.testDatasetA]
self.assertEqual(len(subInfoA), 4, "Dataset A was not subscribed to all sites")
for subInfo in subInfoA:
site = subInfo["node"]
if subInfo['level'] == 'block':
self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "y", "Wrong subscription type for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["node"], "T0_CH_CERN_MSS", "Wrong node for dataset A, block subscription.")
self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription")
elif site == "T0_CH_CERN_MSS":
self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["level"], "dataset", "Wrong level for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription")
elif site == "T1_US_FNAL_MSS":
self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["level"], "dataset", "Wrong level for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription")
elif site == "T3_CO_Uniandes":
self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["level"], "dataset", "Wrong level for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription")
else:
self.fail("Dataset A was subscribed to a wrong site %s" % site)
self.assertEqual(len([x for x in subInfoA if x['level'] == 'block']), 1)
# Now check /BogusPrimary/CRUZET11-v1/RAW
# According to the spec, custodial to T0_CH_CERN_MSS and T1_UK_RAL_MSS
# Request only at both sites and with high priority
self.assertTrue(self.testDatasetB in subscriptions)
subInfoB = subscriptions[self.testDatasetB]
self.assertEqual(len(subInfoB), 2, "Dataset B was not subscribed to all sites")
for subInfo in subInfoB:
site = subInfo["node"]
if site == "T0_CH_CERN_MSS" or site == "T1_UK_RAL_MSS":
self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset B at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset B at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset B at %s" % subInfo["node"])
self.assertEqual(subInfo["level"], "dataset", "Wrong level for dataset B at %s" % subInfo["node"])
self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription")
else:
self.fail("Dataset B was subscribed to a wrong site %s" % site)
myThread = threading.currentThread()
result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset_subscription where subscribed = 1")[0].fetchall()
self.assertEqual(result[0][0], 5, "Custodial move datasets were marked as subscribed")
# Delete the wmbs entries
myThread.dbi.processData("DELETE FROM wmbs_workflow")
subscriber.algorithm({})
self.assertEqual(len(subscriptions[self.testDatasetA]), 5)
self.assertTrue(self.testDatasetB in subscriptions)
self.assertEqual(len(subscriptions[self.testDatasetB]), 3)
self.assertEqual(len([x for x in subscriptions[self.testDatasetB] if x['level'] == 'block']), 1)
return
class WMAgentTest(unittest.TestCase):
"""
_WMAgentTest_
Global unittest for all WMAgent components
"""
# This is an integration test
__integration__ = "Any old bollocks"
sites = ['T2_US_Florida', 'T2_US_UCSD', 'T2_TW_Taiwan', 'T1_CH_CERN']
components = [
'JobCreator', 'JobSubmitter', 'JobTracker', 'JobAccountant',
'JobArchiver', 'TaskArchiver', 'RetryManager', 'ErrorHandler'
]
def setUp(self):
"""
_setUp_
Set up vital components
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=[
"WMCore.WMBS", 'WMCore.MsgService', 'WMCore.ResourceControl',
'WMCore.ThreadPool', 'WMCore.Agent.Database'
],
useDefault=False)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
locationAction = self.daoFactory(classname="Locations.New")
pendingSlots = self.daoFactory(classname="Locations.SetPendingSlots")
for site in self.sites:
locationAction.execute(siteName=site,
seName='se.%s' % (site),
ceName=site)
pendingSlots.execute(siteName=site, pendingSlots=1000)
#Create sites in resourceControl
resourceControl = ResourceControl()
for site in self.sites:
resourceControl.insertSite(siteName=site,
seName='se.%s' % (site),
ceName=site)
resourceControl.insertThreshold(siteName = site, taskType = 'Processing', \
maxSlots = 10000, pendingSlots = 10000)
self.testDir = self.testInit.generateWorkDir()
# Set heartbeat
for component in self.components:
heartbeatAPI = HeartbeatAPI(component)
heartbeatAPI.registerComponent()
self.configFile = EmulatorSetup.setupWMAgentConfig()
return
def tearDown(self):
"""
_tearDown_
Tear down everything and go home.
"""
self.testInit.clearDatabase()
self.testInit.delWorkDir()
EmulatorSetup.deleteConfig(self.configFile)
return
def createTestWorkload(self, workloadName='Test', emulator=True):
"""
_createTestWorkload_
Creates a test workload for us to run on, hold the basic necessities.
"""
workload = testWorkload("TestWorkload")
rereco = workload.getTask("ReReco")
taskMaker = TaskMaker(workload,
os.path.join(self.testDir, 'workloadTest'))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
workload.save(workloadName)
return workload
def getConfig(self):
"""
_getConfig_
This is the global test configuration object
"""
config = self.testInit.getConfiguration()
config.component_("Agent")
config.Agent.WMSpecDirectory = self.testDir
config.Agent.agentName = 'testAgent'
config.Agent.componentName = 'test'
# First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
# Now the CoreDatabase information
# This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# JobCreator
config.component_("JobCreator")
config.JobCreator.namespace = 'WMComponent.JobCreator.JobCreator'
config.JobCreator.logLevel = 'DEBUG'
config.JobCreator.maxThreads = 1
config.JobCreator.UpdateFromResourceControl = True
config.JobCreator.pollInterval = 10
config.JobCreator.jobCacheDir = self.testDir
config.JobCreator.defaultJobType = 'processing' #Type of jobs that we run, used for resource control
config.JobCreator.workerThreads = 2
config.JobCreator.componentDir = os.path.join(os.getcwd(),
'Components')
# JobSubmitter
config.component_("JobSubmitter")
config.JobSubmitter.namespace = 'WMComponent.JobSubmitter.JobSubmitter'
config.JobSubmitter.logLevel = 'INFO'
config.JobSubmitter.maxThreads = 1
config.JobSubmitter.pollInterval = 10
config.JobSubmitter.pluginName = 'CondorGlobusPlugin'
config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins'
config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit')
config.JobSubmitter.submitNode = os.getenv("HOSTNAME",
'badtest.fnal.gov')
config.JobSubmitter.submitScript = os.path.join(
getWMBASE(), 'test/python/WMComponent_t/JobSubmitter_t',
'submit.sh')
config.JobSubmitter.componentDir = os.path.join(
os.getcwd(), 'Components')
config.JobSubmitter.workerThreads = 2
config.JobSubmitter.jobsPerWorker = 200
# JobTracker
config.component_("JobTracker")
config.JobTracker.logLevel = 'DEBUG'
config.JobTracker.pollInterval = 10
config.JobTracker.trackerName = 'CondorTracker'
config.JobTracker.pluginDir = 'WMComponent.JobTracker.Plugins'
config.JobTracker.componentDir = os.path.join(os.getcwd(),
'Components')
config.JobTracker.runTimeLimit = 7776000 #Jobs expire after 90 days
config.JobTracker.idleTimeLimit = 7776000
config.JobTracker.heldTimeLimit = 7776000
config.JobTracker.unknTimeLimit = 7776000
# JobAccountant
config.component_("JobAccountant")
config.JobAccountant.pollInterval = 60
config.JobAccountant.componentDir = os.path.join(
os.getcwd(), 'Components')
config.JobAccountant.logLevel = 'INFO'
# JobArchiver
config.component_("JobArchiver")
config.JobArchiver.pollInterval = 60
config.JobArchiver.logLevel = 'INFO'
config.JobArchiver.logDir = os.path.join(self.testDir, 'logs')
config.JobArchiver.componentDir = os.path.join(os.getcwd(),
'Components')
config.JobArchiver.numberOfJobsToCluster = 1000
# Task Archiver
config.component_("TaskArchiver")
config.TaskArchiver.componentDir = self.testInit.generateWorkDir()
config.TaskArchiver.WorkQueueParams = {}
config.TaskArchiver.pollInterval = 60
config.TaskArchiver.logLevel = 'INFO'
config.TaskArchiver.timeOut = 0
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv(
'COUCHURL', 'mnorman:[email protected]:5984')
config.JobStateMachine.couchDBName = "mnorman_test"
# Needed, because this is a test
os.makedirs(config.JobSubmitter.submitDir)
return config
def createFileCollection(self,
name,
nSubs,
nFiles,
workflowURL='test',
site=None):
"""
_createFileCollection_
Create a collection of files for splitting into jobs
"""
myThread = threading.currentThread()
testWorkflow = Workflow(spec=workflowURL,
owner="mnorman",
name=name,
task="/TestWorkload/ReReco")
testWorkflow.create()
for sub in range(nSubs):
nameStr = '%s-%i' % (name, sub)
testFileset = Fileset(name=nameStr)
testFileset.create()
for f in range(nFiles):
# pick a random site
if not site:
tmpSite = 'se.%s' % (random.choice(self.sites))
else:
tmpSite = 'se.%s' % (site)
testFile = File(lfn="/lfn/%s/%i" % (nameStr, f),
size=1024,
events=10)
testFile.setLocation(tmpSite)
testFile.create()
testFileset.addFile(testFile)
testFileset.commit()
testFileset.markOpen(isOpen=0)
testSubscription = Subscription(fileset=testFileset,
workflow=testWorkflow,
type="Processing",
split_algo="FileBased")
testSubscription.create()
return
def createReports(self, jobs, retryCount=0):
"""
_createReports_
Create some dummy job reports for each job
"""
report = Report()
report.addStep('testStep', 0)
for job in jobs:
#reportPath = os.path.join(job['cache_dir'], 'Report.%i.pkl' % (retryCount))
reportPath = job['fwjr_path']
if os.path.exists(reportPath):
os.remove(reportPath)
report.save(reportPath)
return
def testA_StraightThrough(self):
"""
_StraightThrough_
Just run everything straight through without any variations
"""
# Do pre-submit job check
nRunning = getCondorRunningJobs()
self.assertEqual(
nRunning, 0,
"User currently has %i running jobs. Test will not continue" %
(nRunning))
myThread = threading.currentThread()
workload = self.createTestWorkload()
config = self.getConfig()
name = 'WMAgent_Test1'
site = self.sites[0]
nSubs = 5
nFiles = 10
workloadPath = os.path.join(self.testDir, 'workloadTest',
'TestWorkload', 'WMSandbox',
'WMWorkload.pkl')
# Create a collection of files
self.createFileCollection(name=name,
nSubs=nSubs,
nFiles=nFiles,
workflowURL=workloadPath,
site=site)
############################################################
# Test the JobCreator
config.Agent.componentName = 'JobCreator'
testJobCreator = JobCreatorPoller(config=config)
testJobCreator.algorithm()
time.sleep(5)
# Did all jobs get created?
getJobsAction = self.daoFactory(classname="Jobs.GetAllJobs")
result = getJobsAction.execute(state='Created', jobType="Processing")
self.assertEqual(len(result), nSubs * nFiles)
# Count database objects
result = myThread.dbi.processData(
'SELECT * FROM wmbs_sub_files_acquired')[0].fetchall()
self.assertEqual(len(result), nSubs * nFiles)
# Find the test directory
testDirectory = os.path.join(self.testDir, 'TestWorkload', 'ReReco')
self.assertTrue('JobCollection_1_0' in os.listdir(testDirectory))
self.assertTrue(len(os.listdir(testDirectory)) <= 20)
groupDirectory = os.path.join(testDirectory, 'JobCollection_1_0')
# First job should be in here
self.assertTrue('job_1' in os.listdir(groupDirectory))
jobFile = os.path.join(groupDirectory, 'job_1', 'job.pkl')
self.assertTrue(os.path.isfile(jobFile))
f = open(jobFile, 'r')
job = cPickle.load(f)
f.close()
self.assertEqual(job['workflow'], name)
self.assertEqual(len(job['input_files']), 1)
self.assertEqual(os.path.basename(job['sandbox']),
'TestWorkload-Sandbox.tar.bz2')
###############################################################
# Now test the JobSubmitter
config.Agent.componentName = 'JobSubmitter'
testJobSubmitter = JobSubmitterPoller(config=config)
testJobSubmitter.algorithm()
# Check that jobs are in the right state
result = getJobsAction.execute(state='Created', jobType="Processing")
self.assertEqual(len(result), 0)
result = getJobsAction.execute(state='Executing', jobType="Processing")
self.assertEqual(len(result), nSubs * nFiles)
# Check assigned locations
getLocationAction = self.daoFactory(classname="Jobs.GetLocation")
for id in result:
loc = getLocationAction.execute(jobid=id)
self.assertEqual(loc, [[site]])
# Check to make sure we have running jobs
nRunning = getCondorRunningJobs()
self.assertEqual(nRunning, nFiles * nSubs)
#################################################################
# Now the JobTracker
config.Agent.componentName = 'JobTracker'
testJobTracker = JobTrackerPoller(config=config)
testJobTracker.setup()
testJobTracker.algorithm()
# Running the algo without removing the jobs should do nothing
result = getJobsAction.execute(state='Executing', jobType="Processing")
self.assertEqual(len(result), nSubs * nFiles)
condorRM()
time.sleep(1)
# All jobs gone?
nRunning = getCondorRunningJobs()
self.assertEqual(nRunning, 0)
testJobTracker.algorithm()
time.sleep(5)
# Running the algo without removing the jobs should do nothing
result = getJobsAction.execute(state='Executing', jobType="Processing")
self.assertEqual(len(result), 0)
result = getJobsAction.execute(state='Complete', jobType="Processing")
self.assertEqual(len(result), nSubs * nFiles)
#################################################################
# Now the JobAccountant
# First you need to load all jobs
self.getFWJRAction = self.daoFactory(classname="Jobs.GetFWJRByState")
completeJobs = self.getFWJRAction.execute(state="complete")
# Create reports for all jobs
self.createReports(jobs=completeJobs, retryCount=0)
config.Agent.componentName = 'JobAccountant'
testJobAccountant = JobAccountantPoller(config=config)
testJobAccountant.setup()
# It should do something with the jobs
testJobAccountant.algorithm()
# All the jobs should be done now
result = getJobsAction.execute(state='Complete', jobType="Processing")
self.assertEqual(len(result), 0)
result = getJobsAction.execute(state='Success', jobType="Processing")
self.assertEqual(len(result), nSubs * nFiles)
#######################################################################
# Now the JobArchiver
config.Agent.componentName = 'JobArchiver'
testJobArchiver = JobArchiverPoller(config=config)
testJobArchiver.algorithm()
# All the jobs should be cleaned up
result = getJobsAction.execute(state='Success', jobType="Processing")
self.assertEqual(len(result), 0)
result = getJobsAction.execute(state='Cleanout', jobType="Processing")
self.assertEqual(len(result), nSubs * nFiles)
logDir = os.path.join(self.testDir, 'logs')
for job in completeJobs:
self.assertFalse(os.path.exists(job['fwjr_path']))
jobFolder = 'JobCluster_%i' \
% (int(job['id']/config.JobArchiver.numberOfJobsToCluster))
jobPath = os.path.join(logDir, jobFolder,
'Job_%i.tar' % (job['id']))
self.assertTrue(os.path.isfile(jobPath))
self.assertTrue(os.path.getsize(jobPath) > 0)
###########################################################################
# Now the TaskAchiver
config.Agent.componentName = 'TaskArchiver'
testTaskArchiver = TaskArchiverPoller(config=config)
testTaskArchiver.algorithm()
result = getJobsAction.execute(state='Cleanout', jobType="Processing")
self.assertEqual(len(result), 0)
for jdict in completeJobs:
job = Job(id=jdict['id'])
self.assertFalse(job.exists())
if os.path.isdir('testDir'):
shutil.rmtree('testDir')
shutil.copytree('%s' % self.testDir,
os.path.join(os.getcwd(), 'testDir'))
return
class PhEDExInjectorSubscriberTest(unittest.TestCase):
"""
_PhEDExInjectorSubscriberTest_
Unit tests for the PhEDExInjectorSubscriber.
Create some database inside DBSBuffer, run the subscriber algorithm
using a PhEDEx emulator and verify that it works both in unsafe and safe mode.
For unsafe mode there a WMBS database is also created
"""
def setUp(self):
"""
_setUp_
Install the DBSBuffer schema into the database and connect to PhEDEx.
"""
self.phedexURL = "https://bogus.cern.ch/bogus"
self.dbsURL = "https://bogus.cern.ch/bogus"
EmulatorHelper.setEmulators(phedex = True, dbs = True, siteDB = True)
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer",
"WMCore.WMBS"],
useDefault = False)
self.testFilesA = []
self.testFilesB = []
self.testDatasetA = "/BogusPrimary/Run2012Z-PromptReco-v1/RECO"
self.testDatasetB = "/BogusPrimary/CRUZET11-v1/RAW"
return
def tearDown(self):
"""
_tearDown_
Delete the database.
"""
self.testInit.clearDatabase()
EmulatorHelper.resetEmulators()
def createConfig(self, safeMode):
"""
_createConfig_
Create a config for the PhEDExInjector, paths to DBS and PhEDEx are dummies because
we are using Emulators
"""
config = self.testInit.getConfiguration()
config.component_("DBSInterface")
config.DBSInterface.globalDBSUrl = self.dbsURL
config.component_("PhEDExInjector")
config.PhEDExInjector.phedexurl = self.phedexURL
config.PhEDExInjector.subscribeDatasets = True
config.PhEDExInjector.group = "Saturn"
config.PhEDExInjector.pollInterval = 30
config.PhEDExInjector.subscribeInterval = 60
config.PhEDExInjector.safeOperationMode = safeMode
return config
def stuffDatabase(self):
"""
_stuffDatabase_
Fill the dbsbuffer with some files and blocks. We'll insert a total
of 5 files spanning two blocks. There will be a total of two datasets
inserted into the database, both from the same workflow.
All files will be already in GLOBAL and in_phedex
"""
myThread = threading.currentThread()
buffer3Factory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
insertWorkflow = buffer3Factory(classname = "InsertWorkflow")
insertWorkflow.execute("BogusRequest", "BogusTask", os.path.join(getTestBase(),
"WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl"))
checksums = {"adler32": "1234", "cksum": "5678"}
testFileA = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileA.setDatasetPath(self.testDatasetA)
testFileA.addRun(Run(2, *[45]))
testFileA.create()
testFileB = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileB.setDatasetPath(self.testDatasetA)
testFileB.addRun(Run(2, *[45]))
testFileB.create()
testFileC = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileC.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileC.setDatasetPath(self.testDatasetA)
testFileC.addRun(Run(2, *[45]))
testFileC.create()
self.testFilesA.append(testFileA)
self.testFilesA.append(testFileB)
self.testFilesA.append(testFileC)
testFileD = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileD.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileD.setDatasetPath(self.testDatasetB)
testFileD.addRun(Run(2, *[45]))
testFileD.create()
testFileE = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileE.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileE.setDatasetPath(self.testDatasetB)
testFileE.addRun(Run(2, *[45]))
testFileE.create()
self.testFilesB.append(testFileD)
self.testFilesB.append(testFileE)
uploadFactory = DAOFactory(package = "WMComponent.DBSUpload.Database",
logger = myThread.logger,
dbinterface = myThread.dbi)
createBlock = uploadFactory(classname = "SetBlockStatus")
self.blockAName = self.testDatasetA + "#" + makeUUID()
self.blockBName = self.testDatasetB + "#" + makeUUID()
createBlock.execute(block = self.blockAName, locations = ["srm-cms.cern.ch"], open_status = 0)
createBlock.execute(block = self.blockBName, locations = ["srm-cms.cern.ch"], open_status = 0)
bufferFactory = DAOFactory(package = "WMComponent.DBSBuffer.Database",
logger = myThread.logger,
dbinterface = myThread.dbi)
setBlock = bufferFactory(classname = "DBSBufferFiles.SetBlock")
setBlock.execute(testFileA["lfn"], self.blockAName)
setBlock.execute(testFileB["lfn"], self.blockAName)
setBlock.execute(testFileC["lfn"], self.blockAName)
setBlock.execute(testFileD["lfn"], self.blockBName)
setBlock.execute(testFileE["lfn"], self.blockBName)
fileStatus = bufferFactory(classname = "DBSBufferFiles.SetStatus")
fileStatus.execute(testFileA["lfn"], "GLOBAL")
fileStatus.execute(testFileB["lfn"], "GLOBAL")
fileStatus.execute(testFileC["lfn"], "GLOBAL")
fileStatus.execute(testFileD["lfn"], "GLOBAL")
fileStatus.execute(testFileE["lfn"], "GLOBAL")
phedexStatus = bufferFactory(classname = "DBSBufferFiles.SetPhEDExStatus")
phedexStatus.execute(testFileA["lfn"], 1)
phedexStatus.execute(testFileB["lfn"], 1)
phedexStatus.execute(testFileC["lfn"], 1)
phedexStatus.execute(testFileD["lfn"], 1)
phedexStatus.execute(testFileE["lfn"], 1)
associateWorkflow = buffer3Factory(classname = "DBSBufferFiles.AssociateWorkflowToFile")
associateWorkflow.execute(testFileA["lfn"], "BogusRequest", "BogusTask")
associateWorkflow.execute(testFileB["lfn"], "BogusRequest", "BogusTask")
associateWorkflow.execute(testFileC["lfn"], "BogusRequest", "BogusTask")
associateWorkflow.execute(testFileD["lfn"], "BogusRequest", "BogusTask")
associateWorkflow.execute(testFileE["lfn"], "BogusRequest", "BogusTask")
return
def stuffWMBS(self):
"""
_stuffWMBS_
Inject the workflow in WMBS and add the subscriptions
"""
testWorkflow = Workflow(spec = os.path.join(getTestBase(),
"WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl"),
owner = "/CN=OU/DN=SomeoneWithPermissions",
name = "BogusRequest", task = "BogusTask", owner_vogroup = "", owner_vorole = "")
testWorkflow.create()
testMergeWorkflow = Workflow(spec = os.path.join(getTestBase(),
"WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl"),
owner = "/CN=OU/DN=SomeoneWithPermissions",
name = "BogusRequest", task = "BogusTask/Merge", owner_vogroup = "", owner_vorole = "")
testMergeWorkflow.create()
testWMBSFileset = Fileset(name = "TopFileset")
testWMBSFileset.create()
testWMBSFilesetUnmerged = Fileset(name = "UnmergedFileset")
testWMBSFilesetUnmerged.create()
testFileA = File(lfn = "/this/is/a/lfnA" , size = 1024, events = 10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10)
testFileB.addRun(Run(10, *[12314]))
testFileB.setLocation('malpaquet')
testFileA.create()
testFileB.create()
testWMBSFileset.addFile(testFileA)
testWMBSFilesetUnmerged.addFile(testFileB)
testWMBSFileset.commit()
testWMBSFilesetUnmerged.commit()
testSubscription = Subscription(fileset = testWMBSFileset,
workflow = testWorkflow)
testSubscription.create()
testSubscriptionMerge = Subscription(fileset = testWMBSFilesetUnmerged,
workflow = testMergeWorkflow,
type = "Merge")
testSubscriptionMerge.create()
return (testSubscription, testSubscriptionMerge)
def testUnsafeModeSubscriptions(self):
"""
_testUnsafeModeSubscriptions_
Tests that we can make custodial/non-custodial subscriptions on
unsafe operation mode, this time we don't need WMBS for anything.
All is subscribed in one go.
Check that the requests are correct.
"""
self.stuffDatabase()
config = self.createConfig(safeMode = False)
subscriber = PhEDExInjectorSubscriber(config)
subscriber.setup({})
subscriber.algorithm({})
phedexInstance = subscriber.phedex
subscriptions = phedexInstance.subRequests
# Let's check /BogusPrimary/Run2012Z-PromptReco-v1/RECO
# According to the spec, this should be custodial at T1_US_FNAL
# Non-custodial at T1_UK_RAL and T3_CO_Uniandes
# Autoapproved in all non-custodial sites
# Priority is normal
self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed")
subInfoA = subscriptions[self.testDatasetA]
self.assertEqual(len(subInfoA), 3, "Dataset A was not subscribed to all sites")
for subInfo in subInfoA:
site = subInfo["node"][0]
self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription")
if site == "T1_UK_RAL_MSS" or site == "T3_CO_Uniandes":
self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"])
elif site == "T1_US_FNAL_MSS":
self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "y", "Wrong subscription type for dataset A at %s" % subInfo["node"])
else:
self.fail("Dataset A was subscribed to a wrong site %s" % site)
# Now check /BogusPrimary/CRUZET11-v1/RAW
# According to the spec, this is not custodial anywhere
# Non-custodial at T1_UK_RAL and T2_CH_CERN
# Request only at both sites and with high priority
self.assertTrue(self.testDatasetB in subscriptions, "Dataset B was not subscribed")
subInfoB = subscriptions[self.testDatasetB]
self.assertEqual(len(subInfoB), 2, "Dataset B was not subscribed to all sites")
for subInfo in subInfoB:
site = subInfo["node"][0]
self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription")
if site == "T1_UK_RAL_MSS" or site == "T2_CH_CERN":
self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset B at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset B at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset B at %s" % subInfo["node"])
else:
self.fail("Dataset B was subscribed to a wrong site %s" % site)
myThread = threading.currentThread()
result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset where subscribed = 1")[0].fetchall()
self.assertEqual(result[0][0], 2, "Not all datasets were marked as subscribed")
return
def testSafeModeSubscriptions(self):
"""
_testSafeModeSubscriptions_
Tests that we can make custodial/non-custodial subscriptions on
safe operation mode, make sure that the flow of subscriptions
obeys the rule laid in the subscriber documentation.
Check that the requests are correct.
"""
config = self.createConfig(safeMode = True)
self.stuffDatabase()
topSubscription, mergeSubscription = self.stuffWMBS()
# Start the subscriber
subscriber = PhEDExInjectorSubscriber(config)
subscriber.setup({})
# Run once, this means that all custodial and non-custodial subscriptions
# will be made but none will be Move
subscriber.algorithm({})
phedexInstance = subscriber.phedex
subscriptions = phedexInstance.subRequests
# Let's check /BogusPrimary/Run2012Z-PromptReco-v1/RECO
# According to the spec, this should be custodial at T1_US_FNAL
# Non-custodial at T1_UK_RAL and T3_CO_Uniandes
# Autoapproved in all non-custodial sites
# Priority is normal
self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed")
subInfoA = subscriptions[self.testDatasetA]
self.assertEqual(len(subInfoA), 3, "Dataset A was not subscribed to all sites")
for subInfo in subInfoA:
site = subInfo["node"][0]
self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription")
if site == "T1_UK_RAL_MSS" or site == "T3_CO_Uniandes":
self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"])
elif site == "T1_US_FNAL_MSS":
self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"])
else:
self.fail("Dataset A was subscribed to a wrong site %s" % site)
# Now check /BogusPrimary/CRUZET11-v1/RAW
# According to the spec, this is not custodial anywhere
# Non-custodial at T1_UK_RAL and T2_CH_CERN
# Request only at both sites and with high priority
self.assertTrue(self.testDatasetB in subscriptions, "Dataset B was not subscribed")
subInfoB = subscriptions[self.testDatasetB]
self.assertEqual(len(subInfoB), 2, "Dataset B was not subscribed to all sites")
for subInfo in subInfoB:
site = subInfo["node"][0]
self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription")
if site == "T1_UK_RAL_MSS" or site == "T2_CH_CERN":
self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset B at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset B at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset B at %s" % subInfo["node"])
else:
self.fail("Dataset B was subscribed to a wrong site %s" % site)
myThread = threading.currentThread()
result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset where subscribed = 1")[0].fetchall()
self.assertEqual(result[0][0], 2, "Not all datasets were marked as partially subscribed")
# Now finish the Processing subscription and run the algorithm again
topSubscription.markFinished()
subscriber.algorithm({})
self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed")
subInfoA = subscriptions[self.testDatasetA]
self.assertEqual(len(subInfoA), 4, "Dataset A was not subscribed again to custodial site")
moveCount = 0
for subInfo in subInfoA:
site = subInfo["node"][0]
self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription")
if site == "T1_UK_RAL_MSS" or site == "T3_CO_Uniandes":
self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"])
elif site == "T1_US_FNAL_MSS":
self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset A at %s" % subInfo["node"])
if subInfo["move"] == "y":
moveCount += 1
else:
self.fail("Dataset A was subscribed to a wrong site %s" % site)
self.assertEqual(moveCount, 1, "Move subscription was not made")
self.assertTrue(self.testDatasetB in subscriptions, "Dataset B was not subscribed")
subInfoB = subscriptions[self.testDatasetB]
self.assertEqual(len(subInfoB), 2, "Dataset B was susbcribed again")
result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset where subscribed = 2")[0].fetchall()
self.assertEqual(result[0][0], 2, "Not all datasets were marked as subscribed")
return
class HarnessTest(unittest.TestCase):
"""
TestCase for TestComponent module
"""
tempDir = None
def setUp(self):
"""
setup for test.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema()
def tearDown(self):
"""
Delete database
"""
self.testInit.clearDatabase()
def testB(self):
raise nose.SkipTest
config = self.testInit.getConfiguration()
self.tempDir = self.testInit.generateWorkDir(config)
config.component_("TestComponent")
config.TestComponent.logLevel = 'INFO'
config.section_("General")
config.TestComponent.componentDir = os.path.join( \
self.tempDir, "Components/TestComponent1")
config.General.workDir = config.TestComponent.componentDir
os.makedirs( config.TestComponent.componentDir )
# as this is a test we build the string from our global environment
# parameters normally you put this straight into the DefaultConfig.py file:
# testInit.getConfiguration returns from the environment variable by default
testComponent = TestComponent(config)
testComponent.prepareToStart()
testComponent.handleMessage('LogState','')
testComponent.handleMessage('TestMessage1','TestMessag1Payload')
testComponent.handleMessage('TestMessage2','TestMessag2Payload')
testComponent.handleMessage('TestMessage3','TestMessag3Payload')
testComponent.handleMessage('TestMessage4','TestMessag4Payload')
testComponent.handleMessage('Logging.DEBUG','')
testComponent.handleMessage('Logging.WARNING','')
testComponent.handleMessage('Logging.CRITICAL','')
testComponent.handleMessage('Logging.ERROR','')
testComponent.handleMessage('Logging.INFO','')
testComponent.handleMessage('Logging.SQLDEBUG','')
testComponent.handleMessage('TestComponent:Logging.DEBUG','')
testComponent.handleMessage('TestComponent:Logging.WARNING','')
testComponent.handleMessage('TestComponent:Logging.CRITICAL','')
testComponent.handleMessage('TestComponent:Logging.ERROR','')
testComponent.handleMessage('TestComponent:Logging.INFO','')
testComponent.handleMessage('TestComponent:Logging.SQLDEBUG','')
# test a non existing message (to generate an error)
errorMsg = ''
try:
testComponent.handleMessage('NonExistingMessageType','')
except Exception as ex:
errorMsg = str(ex)
self.assertTrue(errorMsg.startswith('Message NonExistingMessageType with payload'))
def testC(self):
raise nose.SkipTest
config = self.testInit.getConfiguration()
self.tempDir = self.testInit.generateWorkDir(config)
config.component_("TestComponent")
config.TestComponent.logLevel = 'INFO'
config.section_("General")
# try starting a component as a daemon:
config.TestComponent.componentDir = os.path.join( \
self.tempDir, "Components/TestComponent1")
os.makedirs( config.TestComponent.componentDir )
testComponent = TestComponent(config)
# we set the parent to true as we are testing
testComponent.startDaemon(keepParent = True)
print('trying to kill the component')
time.sleep(2)
daemonFile = os.path.join(config.TestComponent.componentDir, "Daemon.xml")
details = Details(daemonFile)
print('Is component alive: '+str(details.isAlive()))
time.sleep(2)
details.killWithPrejudice()
print('Daemon killed')
def testD(self):
raise nose.SkipTest
config = self.testInit.getConfiguration()
config.component_("TestComponent")
config.TestComponent.logLevel = 'INFO'
config.section_("General")
self.tempDir = self.testInit.generateWorkDir(config)
# try starting a component as a daemon:
config.TestComponent.componentDir = os.path.join( \
self.tempDir, "Components/TestComponent2")
os.makedirs( config.TestComponent.componentDir )
testComponent = TestComponent(config)
# we set the parent to true as we are testing
testComponent.startDaemon(keepParent = True)
time.sleep(2)
daemonFile = os.path.join(config.TestComponent.componentDir, "Daemon.xml")
details = Details(daemonFile)
print('Is component alive: '+str(details.isAlive()))
#create msgService to send stop message.
myThread = threading.currentThread()
factory = WMFactory("msgService", "WMCore.MsgService."+ \
myThread.dialect)
myThread.transaction = Transaction(myThread.dbi)
msgService = factory.loadObject("MsgService")
msgService.registerAs("HarnessTest")
myThread.transaction.commit()
print('Publish a stop message to test if the component shutsdown gracefully')
myThread.transaction.begin()
msg = {'name' : 'Stop', 'payload' : ''}
msgService.publish(msg)
myThread.transaction.commit()
msgService.finish()
while details.isAlive():
print('Component has not received stop message')
time.sleep(2)
print('Daemon shutdown gracefully')
class HarvestTest(unittest.TestCase):
"""
_HarvestTest_
Test for EndOfRun job splitter
"""
def setUp(self):
"""
_setUp_
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMCore.WMBS"])
self.splitterFactory = SplitterFactory(package = "WMCore.JobSplitting")
myThread = threading.currentThread()
self.myThread = myThread
daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = logging,
dbinterface = myThread.dbi)
self.WMBSFactory = daoFactory
config = self.getConfig()
self.changer = ChangeState(config)
myResourceControl = ResourceControl()
myResourceControl.insertSite("SomeSite", 10, 20, "SomeSE", "SomeCE")
myResourceControl.insertSite("SomeSite", 10, 20, "SomeSE2", "SomeCE")
myResourceControl.insertSite("SomeSite2", 10, 20, "SomeSE3", "SomeCE2")
self.fileset1 = Fileset(name = "TestFileset1")
for file in range(11):
newFile = File("/some/file/name%d" % file, size = 1000, events = 100)
newFile.addRun(Run(1,*[1]))
newFile.setLocation('SomeSE')
self.fileset1.addFile(newFile)
self.fileset1.create()
workflow1 = Workflow(spec = "spec.xml", owner = "hufnagel", name = "TestWorkflow1", task="Test")
workflow1.create()
self.subscription1 = Subscription(fileset = self.fileset1,
workflow = workflow1,
split_algo = "Harvest",
type = "Harvesting")
self.subscription1.create()
self.configFile = EmulatorSetup.setupWMAgentConfig()
return
def tearDown(self):
"""
_tearDown_
"""
self.testInit.clearDatabase()
EmulatorSetup.deleteConfig(self.configFile)
return
def getConfig(self):
"""
_getConfig_
"""
config = self.testInit.getConfiguration()
self.testInit.generateWorkDir(config)
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL', None)
config.JobStateMachine.couchDBName = 'wmagent_jobdump'
return config
def finishJobs(self, jobGroups, subscription = None):
"""
_finishJobs_
"""
if not subscription:
subscription = self.subscription1
for f in subscription.acquiredFiles():
subscription.completeFiles(f)
for jobGroup in jobGroups:
self.changer.propagate(jobGroup.jobs, 'executing', 'created')
self.changer.propagate(jobGroup.jobs, 'complete', 'executing')
self.changer.propagate(jobGroup.jobs, 'success', 'complete')
self.changer.propagate(jobGroup.jobs, 'cleanout', 'success')
return
def testHarvestEndOfRunTrigger(self):
"""
_testDQMHarvestEndOfRunTrigger_
Make sure that the basic splitting algo works, which is only, ExpressMerge is ALL done, fire a job against that fileset
"""
self.assertEqual(self.fileset1.open, True, "Fileset is closed. Shouldn't")
jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)
jobGroups = jobFactory()
self.assertEqual(len(jobGroups), 0 , "We got 1 or more jobGroups with an open fileset and no periodic configuration")
self.fileset1.markOpen(False)
self.assertEqual(self.fileset1.open, False, "Fileset is opened, why?")
# We should also check if there are aqcuired files, if there are, there are jobs,
# we don't want to fire another jobs while previous are running (output is integrating whatever input)
# TODO : The above one we can do when all is done. Not priority
jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)
jobGroups = jobFactory()
self.assertEqual(len(jobGroups), 1 , "Harvest jobsplitter didn't create a single jobGroup after the fileset was closed")
return
def testPeriodicTrigger(self):
"""
_testPeriodicTrigger_
"""
self.assertEqual(self.fileset1.open, True, "Fileset is not open, not testing periodic here")
# Test timeout (5s for this first test)
# there should be no acquired files, if there are, shouldn't be a job
#self.subscription1.acquireFiles(self.subscription1.availableFiles().pop())
jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval = 3)
self.assertEqual(len(jobGroups), 1 , "Didn't created the first periodic job when there were acquired files")
# For the whole thing to work, faking the first job finishing, and putting the files as complete
self.finishJobs(jobGroups)
# Adding more of files, so we have new stuff to process
for file in range(12,24):
newFile = File("/some/file/name%d" % file, size = 1000, events = 100)
newFile.addRun(Run(1,*[1]))
newFile.setLocation('SomeSE')
self.fileset1.addFile(newFile)
self.fileset1.commit()
# Testing that it doesn't create a job unless the delay is past
jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval = 2)
self.assertEqual(len(jobGroups), 0 , "Created one or more job, when there were non-acquired file and the period is not passed by")
time.sleep(2)
jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval = 2)
self.assertEqual(len(jobGroups), 1 , "Didn't created one or more job, and there weren't and the period is passed by")
# Finishing out previous jobs
self.finishJobs(jobGroups)
# Adding more of files, so we have new stuff to process
for file in range(26,36):
newFile = File("/some/file/name%d" % file, size = 1000, events = 100)
newFile.addRun(Run(1,*[1]))
newFile.setLocation('SomeSE')
self.fileset1.addFile(newFile)
self.fileset1.commit()
# Trying to create another job just afterwards, it shouldn't, because it should respect the configured delay
jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval = 2)
self.assertEqual(len(jobGroups), 0 , "Created one or more job, there are new files, but the delay is not past")
time.sleep(2)
jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval = 2)
self.assertEqual(len(jobGroups), 1 , "Didn't created one or more job, there are new files and the delay is past")
# Last check is whether the job gets all the files or not
numFilesJob = jobGroups[0].jobs[0].getFiles()
numFilesFileset = self.fileset1.getFiles()
self.assertEqual(numFilesJob, numFilesFileset, "Job didn't got all the files")
# Finishing out previous jobs
self.finishJobs(jobGroups)
# Adding files for the first location
for file in range(38,48):
newFile = File("/some/file/name%d" % file, size = 1000, events = 100)
newFile.addRun(Run(1,*[1]))
newFile.setLocation('SomeSE')
self.fileset1.addFile(newFile)
self.fileset1.commit()
# Then another location
for file in range(50,56):
newFile = File("/some/file/name%d" % file, size = 1000, events = 100)
newFile.addRun(Run(1,*[1]))
newFile.setLocation('SomeSE3')
self.fileset1.addFile(newFile)
self.fileset1.commit()
# We should have jobs in both locations
time.sleep(2)
jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval = 2)
self.assertEqual(len(jobGroups[0].getJobs()), 2 , "We didn't get 2 jobs for 2 locations")
firstJobLocation = jobGroups[0].getJobs()[0].getFileLocations()[0]
secondJobLocation = jobGroups[0].getJobs()[1].getFileLocations()[0]
self.assertEqual(firstJobLocation, 'SomeSite', "First job location is not SomeSite")
self.assertEqual(secondJobLocation, 'SomeSite2', "Second job location is not SomeSite2")
self.finishJobs(jobGroups)
for file in range(60,65):
newFile = File("/some/file/name%d" % file, size = 1000, events = 100)
newFile.addRun(Run(2,*[2]))
newFile.setLocation('SomeSE3')
self.fileset1.addFile(newFile)
self.fileset1.commit()
for file in range(70,75):
newFile = File("/some/file/name%d" % file, size = 1000, events = 100)
newFile.addRun(Run(3,*[3]))
newFile.setLocation('SomeSE3')
self.fileset1.addFile(newFile)
self.fileset1.commit()
time.sleep(2)
jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval = 2)
# This is one of the most "complicated" tests so worth to comment, 4 jobs should be created
# 1 - all previous files from SomeSE and run = 1 (a lot, like ~45)
# 2 - Few files from SomeSE3, Run = 1
# 3 - Few files from SomeSE3, Run = 2
# 4 - Few files from SomeSE3, Run = 3
self.assertEqual(len(jobGroups[0].getJobs()), 4 , "We didn't get 4 jobs for adding 2 different runs to SomeSE3")
return
def testMultipleRunHarvesting(self):
"""
_testMultipleRunHarvesting_
Add some files with multiple runs in each, make sure the jobs
are created by location and run. Verify each job mask afterwards.
Note that in this test run are splitted between sites,
in real life that MUST NOT happen we still don't support that.
"""
multipleFilesFileset = Fileset(name = "TestFileset")
newFile = File("/some/file/test1", size = 1000, events = 100)
newFile.addRun(Run(1,*[1,3,4,5,6,7]))
newFile.addRun(Run(2,*[1,2,4,5,6,7]))
newFile.setLocation('SomeSE')
multipleFilesFileset.addFile(newFile)
newFile = File("/some/file/test2", size = 1000, events = 100)
newFile.addRun(Run(1,*[2,8]))
newFile.addRun(Run(2,*[3,8]))
newFile.setLocation('SomeSE3')
multipleFilesFileset.addFile(newFile)
multipleFilesFileset.create()
harvestingWorkflow = Workflow(spec = "spec.xml",
owner = "hufnagel",
name = "TestWorkflow",
task="Test")
harvestingWorkflow.create()
harvestSub = Subscription(fileset = multipleFilesFileset,
workflow = harvestingWorkflow,
split_algo = "Harvest",
type = "Harvesting")
harvestSub.create()
jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = harvestSub)
jobGroups = jobFactory(periodic_harvest_interval = 2)
self.assertEqual(len(jobGroups), 1, "A single job group was not created")
self.assertEqual(len(jobGroups[0].getJobs()), 4,
"Four jobs were not created")
for job in jobGroups[0].getJobs():
runs = job['mask'].getRunAndLumis()
self.assertEqual(len(runs), 1, "Job has more than one run configured")
possibleLumiPairs = {1 : [[1,1],[3,7],[2,2],[8,8]],
2 : [[1,2],[4,7],[3,3],[8,8]]}
run = runs.keys()[0]
for lumiPair in runs[run]:
self.assertTrue(lumiPair in possibleLumiPairs[run], "Strange lumi pair in the job mask")
self.finishJobs(jobGroups, harvestSub)
newFile = File("/some/file/test3", size = 1000, events = 100)
newFile.addRun(Run(1,*range(9,15)))
newFile.setLocation('SomeSE3')
multipleFilesFileset.addFile(newFile)
multipleFilesFileset.commit()
time.sleep(2)
jobGroups = jobFactory(periodic_harvest_interval = 2)
self.assertEqual(len(jobGroups), 1, "A single job group was not created")
self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created")
for job in jobGroups[0].getJobs():
runs = job['mask'].getRunAndLumis()
self.assertEqual(len(runs), 1, "Job has more than one run configured")
possibleLumiPairs = {1 : [[1,1],[3,7],[2,2],[8,8],[9,14]],
2 : [[1,2],[4,7],[3,3],[8,8]]}
run = runs.keys()[0]
for lumiPair in runs[run]:
self.assertTrue(lumiPair in possibleLumiPairs[run], "Strange lumi pair in the job mask")
harvestingWorkflowSib = Workflow(spec = "spec.xml",
owner = "hufnagel",
name = "TestWorkflowSib",
task="TestSib")
harvestingWorkflowSib.create()
harvestSubSib = Subscription(fileset = multipleFilesFileset,
workflow = harvestingWorkflowSib,
split_algo = "Harvest",
type = "Harvesting")
harvestSubSib.create()
jobFactorySib = self.splitterFactory(package = "WMCore.WMBS", subscription = harvestSubSib)
multipleFilesFileset.markOpen(False)
jobGroups = jobFactorySib(periodic_harvest_sibling = True)
self.assertEqual(len(jobGroups), 0, "A single job group was created")
self.finishJobs(jobGroups, harvestSub)
jobGroups = jobFactorySib(periodic_harvest_sibling = True)
self.assertEqual(len(jobGroups), 1, "A single job group was not created")
self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created")
for job in jobGroups[0].getJobs():
runs = job['mask'].getRunAndLumis()
self.assertEqual(len(runs), 1, "Job has more than one run configured")
possibleLumiPairs = {1 : [[1,1],[3,7],[2,2],[8,8],[9,14]],
2 : [[1,2],[4,7],[3,3],[8,8]]}
run = runs.keys()[0]
for lumiPair in runs[run]:
self.assertTrue(lumiPair in possibleLumiPairs[run], "Strange lumi pair in the job mask")
class HarnessTest(unittest.TestCase):
"""
TestCase for TestComponent module
"""
tempDir = None
def setUp(self):
"""
setup for test.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema()
def tearDown(self):
"""
Delete database
"""
self.testInit.clearDatabase()
def testB(self):
raise nose.SkipTest
config = self.testInit.getConfiguration()
self.tempDir = self.testInit.generateWorkDir(config)
config.component_("TestComponent")
config.TestComponent.logLevel = 'INFO'
config.section_("General")
config.TestComponent.componentDir = os.path.join( \
self.tempDir, "Components/TestComponent1")
config.General.workDir = config.TestComponent.componentDir
os.makedirs(config.TestComponent.componentDir)
# as this is a test we build the string from our global environment
# parameters normally you put this straight into the DefaultConfig.py file:
# testInit.getConfiguration returns from the environment variable by default
testComponent = TestComponent(config)
testComponent.prepareToStart()
testComponent.handleMessage('LogState', '')
testComponent.handleMessage('TestMessage1', 'TestMessag1Payload')
testComponent.handleMessage('TestMessage2', 'TestMessag2Payload')
testComponent.handleMessage('TestMessage3', 'TestMessag3Payload')
testComponent.handleMessage('TestMessage4', 'TestMessag4Payload')
testComponent.handleMessage('Logging.DEBUG', '')
testComponent.handleMessage('Logging.WARNING', '')
testComponent.handleMessage('Logging.CRITICAL', '')
testComponent.handleMessage('Logging.ERROR', '')
testComponent.handleMessage('Logging.INFO', '')
testComponent.handleMessage('Logging.SQLDEBUG', '')
testComponent.handleMessage('TestComponent:Logging.DEBUG', '')
testComponent.handleMessage('TestComponent:Logging.WARNING', '')
testComponent.handleMessage('TestComponent:Logging.CRITICAL', '')
testComponent.handleMessage('TestComponent:Logging.ERROR', '')
testComponent.handleMessage('TestComponent:Logging.INFO', '')
testComponent.handleMessage('TestComponent:Logging.SQLDEBUG', '')
# test a non existing message (to generate an error)
errorMsg = ''
try:
testComponent.handleMessage('NonExistingMessageType', '')
except Exception as ex:
errorMsg = str(ex)
self.assertTrue(errorMsg.startswith('Message NonExistingMessageType with payload'))
def testC(self):
raise nose.SkipTest
config = self.testInit.getConfiguration()
self.tempDir = self.testInit.generateWorkDir(config)
config.component_("TestComponent")
config.TestComponent.logLevel = 'INFO'
config.section_("General")
# try starting a component as a daemon:
config.TestComponent.componentDir = os.path.join( \
self.tempDir, "Components/TestComponent1")
os.makedirs(config.TestComponent.componentDir)
testComponent = TestComponent(config)
# we set the parent to true as we are testing
testComponent.startDaemon(keepParent=True)
print('trying to kill the component')
time.sleep(2)
daemonFile = os.path.join(config.TestComponent.componentDir, "Daemon.xml")
details = Details(daemonFile)
print('Is component alive: ' + str(details.isAlive()))
time.sleep(2)
details.killWithPrejudice()
print('Daemon killed')
def testD(self):
raise nose.SkipTest
config = self.testInit.getConfiguration()
config.component_("TestComponent")
config.TestComponent.logLevel = 'INFO'
config.section_("General")
self.tempDir = self.testInit.generateWorkDir(config)
# try starting a component as a daemon:
config.TestComponent.componentDir = os.path.join( \
self.tempDir, "Components/TestComponent2")
os.makedirs(config.TestComponent.componentDir)
testComponent = TestComponent(config)
# we set the parent to true as we are testing
testComponent.startDaemon(keepParent=True)
time.sleep(2)
daemonFile = os.path.join(config.TestComponent.componentDir, "Daemon.xml")
details = Details(daemonFile)
print('Is component alive: ' + str(details.isAlive()))
# create msgService to send stop message.
myThread = threading.currentThread()
factory = WMFactory("msgService", "WMCore.MsgService." + \
myThread.dialect)
myThread.transaction = Transaction(myThread.dbi)
msgService = factory.loadObject("MsgService")
msgService.registerAs("HarnessTest")
myThread.transaction.commit()
print('Publish a stop message to test if the component shutsdown gracefully')
myThread.transaction.begin()
msg = {'name': 'Stop', 'payload': ''}
msgService.publish(msg)
myThread.transaction.commit()
msgService.finish()
while details.isAlive():
print('Component has not received stop message')
time.sleep(2)
print('Daemon shutdown gracefully')
class RucioInjectorPollerTest(EmulatedUnitTestCase):
"""
Tests for the RucioInjectorPoller component
"""
def setUp(self):
"""
Install the DBSBuffer schema into the database and connect to Rucio.
"""
super(RucioInjectorPollerTest, self).setUp()
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase=True)
self.testInit.setSchema(customModules=["WMComponent.DBS3Buffer"],
useDefault=False)
myThread = threading.currentThread()
daofactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
locationAction = daofactory(classname="DBSBufferFiles.AddLocation")
self.locations = ["T2_CH_CERN", "T1_US_FNAL_Disk"]
for rse in self.locations:
locationAction.execute(siteName=rse)
self.testFilesA = []
self.testFilesB = []
self.testDatasetA = "/SampleA/PromptReco-v1/RECO"
self.testDatasetB = "/SampleB/CRUZET11-v1/RAW"
if PY3:
self.assertItemsEqual = self.assertCountEqual
return
def tearDown(self):
"""
Delete the database.
"""
self.testInit.clearDatabase()
def createConfig(self):
"""
Create a basic configuration for the component
"""
config = self.testInit.getConfiguration()
config.component_("RucioInjector")
config.RucioInjector.pollInterval = 300
config.RucioInjector.pollIntervalRules = 43200
config.RucioInjector.cacheExpiration = 2 * 24 * 60 * 60 # two days
config.RucioInjector.createBlockRules = True
config.RucioInjector.RSEPostfix = False # enable it to append _Test to the RSE names
config.RucioInjector.metaDIDProject = "Production"
config.RucioInjector.containerDiskRuleParams = {"weight": "ddm_quota", "copies": 2, "grouping": "DATASET"}
config.RucioInjector.containerDiskRuleRSEExpr = "(tier=2|tier=1)&cms_type=real&rse_type=DISK"
config.RucioInjector.rucioAccount = "wma_test"
config.RucioInjector.rucioUrl = "http://cmsrucio-int.cern.ch"
config.RucioInjector.rucioAuthUrl = "https://cmsrucio-auth-int.cern.ch"
return config
def stuffDatabase(self):
"""
Fill the dbsbuffer tables with some files and blocks. We'll insert a total
of 5 files spanning two blocks. There will be a total of two datasets
inserted into the database.
We'll inject files with the location set as an SE name as well as a
PhEDEx node name as well.
"""
myThread = threading.currentThread()
# Create the DAOs factory and the relevant instances
buffer3Factory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
setBlock = buffer3Factory(classname="DBSBufferFiles.SetBlock")
fileStatus = buffer3Factory(classname="DBSBufferFiles.SetStatus")
associateWorkflow = buffer3Factory(classname="DBSBufferFiles.AssociateWorkflowToFile")
insertWorkflow = buffer3Factory(classname="InsertWorkflow")
datasetAction = buffer3Factory(classname="NewDataset")
createAction = buffer3Factory(classname="CreateBlocks")
# Create workflow in the database
insertWorkflow.execute("BogusRequest", "BogusTask", 0, 0, 0, 0)
# First file on first block
checksums = {"adler32": "1234", "cksum": "5678"}
testFileA = DBSBufferFile(lfn=makeUUID(), size=1024, events=10,
checksums=checksums,
locations=set(["T2_CH_CERN"]))
testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8",
appFam="RECO", psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFileA.setDatasetPath(self.testDatasetA)
testFileA.addRun(Run(2, *[45]))
testFileA.create()
# Second file on first block
testFileB = DBSBufferFile(lfn=makeUUID(), size=1024, events=10,
checksums=checksums,
locations=set(["T2_CH_CERN"]))
testFileB.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8",
appFam="RECO", psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFileB.setDatasetPath(self.testDatasetA)
testFileB.addRun(Run(2, *[45]))
testFileB.create()
# Third file on first block
testFileC = DBSBufferFile(lfn=makeUUID(), size=1024, events=10,
checksums=checksums,
locations=set(["T2_CH_CERN"]))
testFileC.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8",
appFam="RECO", psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFileC.setDatasetPath(self.testDatasetA)
testFileC.addRun(Run(2, *[45]))
testFileC.create()
self.testFilesA.append(testFileA)
self.testFilesA.append(testFileB)
self.testFilesA.append(testFileC)
# First file on second block
testFileD = DBSBufferFile(lfn=makeUUID(), size=1024, events=10,
checksums=checksums,
locations=set(["T1_US_FNAL_Disk"]))
testFileD.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8",
appFam="RECO", psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFileD.setDatasetPath(self.testDatasetB)
testFileD.addRun(Run(2, *[45]))
testFileD.create()
# Second file on second block
testFileE = DBSBufferFile(lfn=makeUUID(), size=1024, events=10,
checksums=checksums,
locations=set(["T1_US_FNAL_Disk"]))
testFileE.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8",
appFam="RECO", psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFileE.setDatasetPath(self.testDatasetB)
testFileE.addRun(Run(2, *[45]))
testFileE.create()
self.testFilesB.append(testFileD)
self.testFilesB.append(testFileE)
# insert datasets in the dbsbuffer table
datasetAction.execute(datasetPath=self.testDatasetA)
datasetAction.execute(datasetPath=self.testDatasetB)
self.blockAName = self.testDatasetA + "#" + makeUUID()
self.blockBName = self.testDatasetB + "#" + makeUUID()
# create and insert blocks into dbsbuffer table
newBlockA = DBSBufferBlock(name=self.blockAName,
location="T2_CH_CERN",
datasetpath=None)
newBlockA.setDataset(self.testDatasetA, 'data', 'VALID')
newBlockA.status = 'Closed'
newBlockB = DBSBufferBlock(name=self.blockBName,
location="T1_US_FNAL_Disk",
datasetpath=None)
newBlockB.setDataset(self.testDatasetB, 'data', 'VALID')
newBlockB.status = 'Closed'
createAction.execute(blocks=[newBlockA, newBlockB])
# associate files to their correspondent block id
setBlock.execute(testFileA["lfn"], self.blockAName)
setBlock.execute(testFileB["lfn"], self.blockAName)
setBlock.execute(testFileC["lfn"], self.blockAName)
setBlock.execute(testFileD["lfn"], self.blockBName)
setBlock.execute(testFileE["lfn"], self.blockBName)
# set file status to LOCAL
fileStatus.execute(testFileA["lfn"], "LOCAL")
fileStatus.execute(testFileB["lfn"], "LOCAL")
fileStatus.execute(testFileC["lfn"], "LOCAL")
fileStatus.execute(testFileD["lfn"], "LOCAL")
fileStatus.execute(testFileE["lfn"], "LOCAL")
# associate files to a given workflow
associateWorkflow.execute(testFileA["lfn"], "BogusRequest", "BogusTask")
associateWorkflow.execute(testFileB["lfn"], "BogusRequest", "BogusTask")
associateWorkflow.execute(testFileC["lfn"], "BogusRequest", "BogusTask")
associateWorkflow.execute(testFileD["lfn"], "BogusRequest", "BogusTask")
associateWorkflow.execute(testFileE["lfn"], "BogusRequest", "BogusTask")
return
def testBadConfig(self):
"""
Test wrong component configuration
"""
config = self.createConfig()
config.RucioInjector.metaDIDProject = "Very invalid project name"
with self.assertRaises(RucioInjectorException):
RucioInjectorPoller(config)
def testActivityMap(self):
"""
Initialize a RucioInjectorPoller object and test `_activityMap` method
"""
poller = RucioInjectorPoller(self.createConfig())
# test production agent and non-Tape endpoint
activity = poller._activityMap("T1_US_FNAL_Disk")
self.assertEquals(activity, "Production Output")
activity = poller._activityMap("T1_US_FNAL_Test")
self.assertEquals(activity, "Production Output")
# test production agent and Tape endpoint (which is forbidden at the moment)
with self.assertRaises(WMRucioException):
poller._activityMap("T1_US_FNAL_Tape")
# now pretend it to be a T0 agent/component
poller.isT0agent = True
# test T0 agent and non-Tape endpoint
activity = poller._activityMap("T1_US_FNAL_Disk")
self.assertEquals(activity, "T0 Export")
activity = poller._activityMap("T1_US_FNAL_Test")
self.assertEquals(activity, "T0 Export")
# test T0 agent and Tape endpoint
activity = poller._activityMap("T1_US_FNAL_Tape")
self.assertEquals(activity, "T0 Tape")
def testLoadingFiles(self):
"""
Initialize a RucioInjectorPoller object and load uninjected files
"""
self.stuffDatabase()
poller = RucioInjectorPoller(self.createConfig())
poller.setup(parameters=None)
uninjectedFiles = poller.getUninjected.execute()
self.assertItemsEqual(list(uninjectedFiles), self.locations)
self.assertEquals(list(uninjectedFiles["T2_CH_CERN"]), [self.testDatasetA])
self.assertEquals(list(uninjectedFiles["T1_US_FNAL_Disk"]), [self.testDatasetB])
class ProcessPoolTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase = True)
self.testInit.setSchema(customModules = ["WMCore.Agent.Database"],
useDefault = False)
return
def tearDown(self):
"""
_tearDown_
"""
self.testInit.clearDatabase()
return
def testA_ProcessPool(self):
"""
_testProcessPool_
"""
raise nose.SkipTest
config = self.testInit.getConfiguration()
config.Agent.useHeartbeat = False
self.testInit.generateWorkDir(config)
processPool = ProcessPool("ProcessPool_t.ProcessPoolTestWorker",
totalSlaves = 1,
componentDir = config.General.workDir,
config = config,
namespace = "WMCore_t")
processPool.enqueue(["One", "Two", "Three"])
result = processPool.dequeue(3)
self.assertEqual(len(result), 3, "Error: Expected three items back.")
self.assertTrue( "One" in result)
self.assertTrue( "Two" in result)
self.assertTrue( "Three" in result)
return
def testB_ProcessPoolStress(self):
"""
_testProcessPoolStress_
"""
raise nose.SkipTest
config = self.testInit.getConfiguration()
config.Agent.useHeartbeat = False
self.testInit.generateWorkDir(config)
processPool = ProcessPool("ProcessPool_t.ProcessPoolTestWorker",
totalSlaves = 1,
componentDir = config.General.workDir,
namespace = "WMCore_t",
config = config)
result = None
input = None
for i in range(1000):
input = []
while i > 0:
input.append("COMMAND%s" % i)
i -= 1
processPool.enqueue(input)
result = processPool.dequeue(len(input))
self.assertEqual(len(result), len(input),
"Error: Wrong number of results returned.")
for k in result:
self.assertTrue(k in input)
return
def testC_MultiPool(self):
"""
_testMultiPool_
Run a test with multiple workers
"""
raise nose.SkipTest
config = self.testInit.getConfiguration()
config.Agent.useHeartbeat = False
self.testInit.generateWorkDir(config)
processPool = ProcessPool("ProcessPool_t.ProcessPoolTestWorker",
totalSlaves = 3,
componentDir = config.General.workDir,
namespace = "WMCore_t",
config = config)
for i in range(100):
input = []
while i > 0:
input.append("COMMAND%s" % i)
i -= 1
processPool.enqueue(input)
result = processPool.dequeue(len(input))
self.assertEqual(len(result), len(input),
"Error: Wrong number of results returned.")
class ReportIntegrationTest(unittest.TestCase):
"""
_ReportIntegrationTest_
"""
def setUp(self):
"""
_setUp_
Setup the database and WMBS for the test.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer",
"WMCore.WMBS"],
useDefault = False)
myThread = threading.currentThread()
self.daofactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.dbsfactory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = self.daofactory(classname = "Locations.New")
locationAction.execute(siteName = "site1", pnn = "T1_US_FNAL_Disk")
inputFile = File(lfn = "/path/to/some/lfn", size = 10, events = 10,
locations = "T1_US_FNAL_Disk")
inputFile.create()
inputFileset = Fileset(name = "InputFileset")
inputFileset.create()
inputFileset.addFile(inputFile)
inputFileset.commit()
unmergedFileset = Fileset(name = "UnmergedFileset")
unmergedFileset.create()
mergedFileset = Fileset(name = "MergedFileset")
mergedFileset.create()
procWorkflow = Workflow(spec = "wf001.xml", owner = "Steve",
name = "TestWF", task = "/TestWF/None")
procWorkflow.create()
procWorkflow.addOutput("outputRECORECO", unmergedFileset)
mergeWorkflow = Workflow(spec = "wf002.xml", owner = "Steve",
name = "MergeWF", task = "/MergeWF/None")
mergeWorkflow.create()
mergeWorkflow.addOutput("Merged", mergedFileset)
insertWorkflow = self.dbsfactory(classname = "InsertWorkflow")
insertWorkflow.execute("TestWF", "/TestWF/None", 0, 0, 0, 0)
insertWorkflow.execute("MergeWF", "/MergeWF/None", 0, 0, 0, 0)
self.procSubscription = Subscription(fileset = inputFileset,
workflow = procWorkflow,
split_algo = "FileBased",
type = "Processing")
self.procSubscription.create()
self.procSubscription.acquireFiles()
self.mergeSubscription = Subscription(fileset = unmergedFileset,
workflow = mergeWorkflow,
split_algo = "WMBSMergeBySize",
type = "Merge")
self.mergeSubscription.create()
self.procJobGroup = JobGroup(subscription = self.procSubscription)
self.procJobGroup.create()
self.mergeJobGroup = JobGroup(subscription = self.mergeSubscription)
self.mergeJobGroup.create()
self.testJob = Job(name = "testJob", files = [inputFile])
self.testJob.create(group = self.procJobGroup)
self.testJob["state"] = "complete"
myThread = threading.currentThread()
self.daofactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.stateChangeAction = self.daofactory(classname = "Jobs.ChangeState")
self.setFWJRAction = self.daofactory(classname = "Jobs.SetFWJRPath")
self.getJobTypeAction = self.daofactory(classname = "Jobs.GetType")
locationAction = self.daofactory(classname = "Locations.New")
locationAction.execute(siteName = "cmssrm.fnal.gov")
self.stateChangeAction.execute(jobs = [self.testJob])
self.tempDir = tempfile.mkdtemp()
return
def tearDown(self):
"""
_tearDown_
Clear out the database and the pickled report file.
"""
self.testInit.clearDatabase()
try:
os.remove(os.path.join(self.tempDir, "ProcReport.pkl"))
os.remove(os.path.join(self.tempDir, "MergeReport.pkl"))
except Exception as ex:
pass
try:
os.rmdir(self.tempDir)
except Exception as ex:
pass
return
def createConfig(self, workerThreads):
"""
_createConfig_
Create a config for the JobAccountant with the given number of worker
threads. This config needs to include information for connecting to the
database as the component will create it's own database connections.
These parameters are still pulled from the environment.
"""
config = self.testInit.getConfiguration()
self.testInit.generateWorkDir(config)
config.section_("JobStateMachine")
config.JobStateMachine.couchurl = os.getenv("COUCHURL")
config.JobStateMachine.couchDBName = "report_integration_t"
config.JobStateMachine.jobSummaryDBName = "report_integration_wmagent_summary_t"
config.component_("JobAccountant")
config.JobAccountant.pollInterval = 60
config.JobAccountant.workerThreads = workerThreads
config.JobAccountant.componentDir = os.getcwd()
config.JobAccountant.logLevel = 'SQLDEBUG'
config.component_("TaskArchiver")
config.TaskArchiver.localWMStatsURL = "%s/%s" % (config.JobStateMachine.couchurl, config.JobStateMachine.jobSummaryDBName)
return config
def verifyJobSuccess(self, jobID):
"""
_verifyJobSuccess_
Verify that the metadata for a successful job is correct. This will
check the outcome, retry count and state.
"""
testJob = Job(id = jobID)
testJob.load()
assert testJob["state"] == "success", \
"Error: test job in wrong state: %s" % testJob["state"]
assert testJob["retry_count"] == 0, \
"Error: test job has wrong retry count: %s" % testJob["retry_count"]
assert testJob["outcome"] == "success", \
"Error: test job has wrong outcome: %s" % testJob["outcome"]
return
def verifyFileMetaData(self, jobID, fwkJobReportFiles):
"""
_verifyFileMetaData_
Verify that all the files that were output by a job made it into WMBS
correctly. Compare the contents of WMBS to the files in the frameworks
job report.
Note that fwkJobReportFiles is a list of DataStructs File objects.
"""
testJob = Job(id = jobID)
testJob.loadData()
inputLFNs = []
for inputFile in testJob["input_files"]:
inputLFNs.append(inputFile["lfn"])
for fwkJobReportFile in fwkJobReportFiles:
outputFile = File(lfn = fwkJobReportFile["lfn"])
outputFile.loadData(parentage = 1)
assert outputFile["events"] == int(fwkJobReportFile["events"]), \
"Error: Output file has wrong events: %s, %s" % \
(outputFile["events"], fwkJobReportFile["events"])
assert outputFile["size"] == int(fwkJobReportFile["size"]), \
"Error: Output file has wrong size: %s, %s" % \
(outputFile["size"], fwkJobReportFile["size"])
for ckType in fwkJobReportFile["checksums"]:
assert ckType in outputFile["checksums"], \
"Error: Output file is missing checksums: %s" % ckType
assert outputFile["checksums"][ckType] == fwkJobReportFile["checksums"][ckType], \
"Error: Checksums don't match."
assert len(fwkJobReportFile["checksums"]) == \
len(outputFile["checksums"]), \
"Error: Wrong number of checksums."
jobType = self.getJobTypeAction.execute(jobID = jobID)
if jobType == "Merge":
assert str(outputFile["merged"]) == "True", \
"Error: Merge jobs should output merged files."
else:
assert outputFile["merged"] == fwkJobReportFile["merged"], \
"Error: Output file merged output is wrong: %s, %s" % \
(outputFile["merged"], fwkJobReportFile["merged"])
assert len(outputFile["locations"]) == 1, \
"Error: outputfile should have one location: %s" % outputFile["locations"]
assert list(outputFile["locations"])[0] == list(fwkJobReportFile["locations"])[0], \
"Error: wrong location for file."
assert len(outputFile["parents"]) == len(inputLFNs), \
"Error: Output file has wrong number of parents."
for outputParent in outputFile["parents"]:
assert outputParent["lfn"] in inputLFNs, \
"Error: Unknown parent file: %s" % outputParent["lfn"]
fwjrRuns = {}
for run in fwkJobReportFile["runs"]:
fwjrRuns[run.run] = run.lumis
for run in outputFile["runs"]:
assert run.run in fwjrRuns, \
"Error: Extra run in output: %s" % run.run
for lumi in run:
assert lumi in fwjrRuns[run.run], \
"Error: Extra lumi: %s" % lumi
fwjrRuns[run.run].remove(lumi)
if len(fwjrRuns[run.run]) == 0:
del fwjrRuns[run.run]
assert len(fwjrRuns) == 0, \
"Error: Missing runs, lumis: %s" % fwjrRuns
testJobGroup = JobGroup(id = testJob["jobgroup"])
testJobGroup.loadData()
jobGroupFileset = testJobGroup.output
jobGroupFileset.loadData()
assert outputFile["id"] in jobGroupFileset.getFiles(type = "id"), \
"Error: output file not in jobgroup fileset."
if testJob["mask"]["FirstEvent"] == None:
assert outputFile["first_event"] == 0, \
"Error: first event not set correctly: 0, %s" % \
outputFile["first_event"]
else:
assert testJob["mask"]["FirstEvent"] == outputFile["first_event"], \
"Error: last event not set correctly: %s, %s" % \
(testJob["mask"]["FirstEvent"], outputFile["first_event"])
return
def testReportHandling(self):
"""
_testReportHandling_
Verify that we're able to parse a CMSSW report, convert it to a Report()
style report, pickle it and then have the accountant process it.
"""
self.procPath = os.path.join(WMCore.WMBase.getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml")
myReport = Report("cmsRun1")
myReport.parse(self.procPath)
# Fake some metadata that should be added by the stageout scripts.
for fileRef in myReport.getAllFileRefsFromStep("cmsRun1"):
fileRef.size = 1024
fileRef.location = "cmssrm.fnal.gov"
fwjrPath = os.path.join(self.tempDir, "ProcReport.pkl")
cmsRunStep = myReport.retrieveStep("cmsRun1")
cmsRunStep.status = 0
myReport.setTaskName('/TestWF/None')
myReport.persist(fwjrPath)
self.setFWJRAction.execute(jobID = self.testJob["id"], fwjrPath = fwjrPath)
pFile = DBSBufferFile(lfn = "/path/to/some/lfn", size = 600000, events = 60000)
pFile.setAlgorithm(appName = "cmsRun", appVer = "UNKNOWN",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
pFile.setDatasetPath("/bogus/dataset/path")
#pFile.addRun(Run(1, *[45]))
pFile.create()
config = self.createConfig(workerThreads = 1)
accountant = JobAccountantPoller(config)
accountant.setup()
accountant.algorithm()
self.verifyJobSuccess(self.testJob["id"])
self.verifyFileMetaData(self.testJob["id"], myReport.getAllFilesFromStep("cmsRun1"))
inputFile = File(lfn = "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR09_R_34X_V5_All_v1/0000/outputRECORECO.root")
inputFile.load()
self.testMergeJob = Job(name = "testMergeJob", files = [inputFile])
self.testMergeJob.create(group = self.mergeJobGroup)
self.testMergeJob["state"] = "complete"
self.stateChangeAction.execute(jobs = [self.testMergeJob])
self.mergePath = os.path.join(WMCore.WMBase.getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWMergeReport.xml")
myReport = Report("mergeReco")
myReport.parse(self.mergePath)
# Fake some metadata that should be added by the stageout scripts.
for fileRef in myReport.getAllFileRefsFromStep("mergeReco"):
fileRef.size = 1024
fileRef.location = "cmssrm.fnal.gov"
fileRef.dataset = {"applicationName": "cmsRun", "applicationVersion": "CMSSW_3_4_2_patch1",
"primaryDataset": "MinimumBias", "processedDataset": "Rereco-v1",
"dataTier": "RECO"}
fwjrPath = os.path.join(self.tempDir, "MergeReport.pkl")
myReport.setTaskName('/MergeWF/None')
cmsRunStep = myReport.retrieveStep("mergeReco")
cmsRunStep.status = 0
myReport.persist(fwjrPath)
self.setFWJRAction.execute(jobID = self.testMergeJob["id"], fwjrPath = fwjrPath)
accountant.algorithm()
self.verifyJobSuccess(self.testMergeJob["id"])
self.verifyFileMetaData(self.testMergeJob["id"], myReport.getAllFilesFromStep("mergeReco"))
return
class PhEDExInjectorSubscriberTest(EmulatedUnitTestCase):
"""
_PhEDExInjectorSubscriberTest_
Create some database inside DBSBuffer, run the subscriber algorithm
using a PhEDEx emulator and verify that it works both in unsafe and safe mode.
For unsafe mode there a WMBS database is also created
"""
def __init__(self, methodName='runTest'):
super(PhEDExInjectorSubscriberTest, self).__init__(methodName=methodName)
self.blockAName = None
self.blockBName = None
def setUp(self):
"""
_setUp_
Install the DBSBuffer schema into the database and connect to PhEDEx.
"""
super(PhEDExInjectorSubscriberTest, self).setUp()
self.phedexURL = "https://bogus.cern.ch/bogus"
self.dbsURL = "https://bogus.cern.ch/bogus"
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=["WMComponent.DBS3Buffer",
"WMCore.WMBS"],
useDefault=False)
self.testFilesA = []
self.testFilesB = []
self.testDatasetA = "/BogusPrimary/Run2012Z-PromptReco-v1/RECO"
self.testDatasetB = "/BogusPrimary/CRUZET11-v1/RAW"
return
def tearDown(self):
"""
_tearDown_
Delete the database.
"""
self.testInit.clearDatabase()
super(PhEDExInjectorSubscriberTest, self).tearDown()
def createConfig(self):
"""
_createConfig_
Create a config for the PhEDExInjector, paths to DBS and PhEDEx are dummies because
we are using Emulators
"""
config = self.testInit.getConfiguration()
config.component_("DBSInterface")
config.DBSInterface.globalDBSUrl = self.dbsURL
config.component_("PhEDExInjector")
config.PhEDExInjector.phedexurl = self.phedexURL
config.PhEDExInjector.subscribeDatasets = True
config.PhEDExInjector.group = "Saturn"
config.PhEDExInjector.pollInterval = 30
config.PhEDExInjector.subscribeInterval = 60
return config
def stuffDatabase(self):
"""
_stuffDatabase_
Fill the dbsbuffer with some files and blocks. We'll insert a total
of 5 files spanning two blocks. There will be a total of two datasets
inserted into the database.
All files will be already in GLOBAL and in_phedex
"""
myThread = threading.currentThread()
buffer3Factory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
insertWorkflow = buffer3Factory(classname="InsertWorkflow")
insertWorkflow.execute("BogusRequestA", "BogusTask",
0, 0, 0, 0)
insertWorkflow.execute("BogusRequestB", "BogusTask",
0, 0, 0, 0)
checksums = {"adler32": "1234", "cksum": "5678"}
testFileA = DBSBufferFile(lfn=makeUUID(), size=1024, events=10,
checksums=checksums,
locations=set(["srm-cms.cern.ch"]))
testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8",
appFam="RECO", psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFileA.setDatasetPath(self.testDatasetA)
testFileA.addRun(Run(2, *[45]))
testFileA.create()
testFileB = DBSBufferFile(lfn=makeUUID(), size=1024, events=10,
checksums=checksums,
locations=set(["srm-cms.cern.ch"]))
testFileB.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8",
appFam="RECO", psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFileB.setDatasetPath(self.testDatasetA)
testFileB.addRun(Run(2, *[45]))
testFileB.create()
testFileC = DBSBufferFile(lfn=makeUUID(), size=1024, events=10,
checksums=checksums,
locations=set(["srm-cms.cern.ch"]))
testFileC.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8",
appFam="RECO", psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFileC.setDatasetPath(self.testDatasetA)
testFileC.addRun(Run(2, *[45]))
testFileC.create()
self.testFilesA.append(testFileA)
self.testFilesA.append(testFileB)
self.testFilesA.append(testFileC)
testFileD = DBSBufferFile(lfn=makeUUID(), size=1024, events=10,
checksums=checksums,
locations=set(["srm-cms.cern.ch"]))
testFileD.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8",
appFam="RECO", psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFileD.setDatasetPath(self.testDatasetB)
testFileD.addRun(Run(2, *[45]))
testFileD.create()
testFileE = DBSBufferFile(lfn=makeUUID(), size=1024, events=10,
checksums=checksums,
locations=set(["srm-cms.cern.ch"]))
testFileE.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8",
appFam="RECO", psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFileE.setDatasetPath(self.testDatasetB)
testFileE.addRun(Run(2, *[45]))
testFileE.create()
self.testFilesB.append(testFileD)
self.testFilesB.append(testFileE)
uploadFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
datasetAction = uploadFactory(classname="NewDataset")
createAction = uploadFactory(classname="CreateBlocks")
datasetAction.execute(datasetPath=self.testDatasetA)
datasetAction.execute(datasetPath=self.testDatasetB)
self.blockAName = self.testDatasetA + "#" + makeUUID()
self.blockBName = self.testDatasetB + "#" + makeUUID()
newBlockA = DBSBufferBlock(name=self.blockAName,
location="srm-cms.cern.ch",
datasetpath=None)
newBlockA.setDataset(self.testDatasetA, 'data', 'VALID')
newBlockA.status = 'Closed'
newBlockB = DBSBufferBlock(name=self.blockBName,
location="srm-cms.cern.ch",
datasetpath=None)
newBlockB.setDataset(self.testDatasetB, 'data', 'VALID')
newBlockB.status = 'Closed'
createAction.execute(blocks=[newBlockA, newBlockB])
bufferFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
setBlock = bufferFactory(classname="DBSBufferFiles.SetBlock")
setBlock.execute(testFileA["lfn"], self.blockAName)
setBlock.execute(testFileB["lfn"], self.blockAName)
setBlock.execute(testFileC["lfn"], self.blockAName)
setBlock.execute(testFileD["lfn"], self.blockBName)
setBlock.execute(testFileE["lfn"], self.blockBName)
fileStatus = bufferFactory(classname="DBSBufferFiles.SetStatus")
fileStatus.execute(testFileA["lfn"], "GLOBAL")
fileStatus.execute(testFileB["lfn"], "GLOBAL")
fileStatus.execute(testFileC["lfn"], "GLOBAL")
fileStatus.execute(testFileD["lfn"], "GLOBAL")
fileStatus.execute(testFileE["lfn"], "GLOBAL")
phedexStatus = bufferFactory(classname="DBSBufferFiles.SetPhEDExStatus")
phedexStatus.execute(testFileA["lfn"], 1)
phedexStatus.execute(testFileB["lfn"], 1)
phedexStatus.execute(testFileC["lfn"], 1)
phedexStatus.execute(testFileD["lfn"], 1)
phedexStatus.execute(testFileE["lfn"], 1)
associateWorkflow = buffer3Factory(classname="DBSBufferFiles.AssociateWorkflowToFile")
associateWorkflow.execute(testFileA["lfn"], "BogusRequestA", "BogusTask")
associateWorkflow.execute(testFileB["lfn"], "BogusRequestA", "BogusTask")
associateWorkflow.execute(testFileC["lfn"], "BogusRequestA", "BogusTask")
associateWorkflow.execute(testFileD["lfn"], "BogusRequestB", "BogusTask")
associateWorkflow.execute(testFileE["lfn"], "BogusRequestB", "BogusTask")
# Make the desired subscriptions
insertSubAction = buffer3Factory(classname="NewSubscription")
datasetA = DBSBufferDataset(path=self.testDatasetA)
datasetB = DBSBufferDataset(path=self.testDatasetB)
workload = WMWorkloadHelper()
workload.load(os.path.join(getTestBase(), 'WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl'))
insertSubAction.execute(datasetA.exists(), workload.getSubscriptionInformation()[self.testDatasetA])
insertSubAction.execute(datasetB.exists(), workload.getSubscriptionInformation()[self.testDatasetB])
return
def testNormalModeSubscriptions(self):
"""
_testNormalModeSubscriptions_
Tests that we can make custodial/non-custodial subscriptions on
normal operation mode, this time we don't need WMBS for anything.
All is subscribed in one go.
Check that the requests are correct.
"""
self.stuffDatabase()
config = self.createConfig()
subscriber = PhEDExInjectorPoller(config)
subscriber.setup({})
subscriber.subscribeDatasets()
phedexInstance = subscriber.phedex
subscriptions = phedexInstance.subRequests
# Let's check /BogusPrimary/Run2012Z-PromptReco-v1/RECO
# According to the spec, this should be custodial at T1_US_FNAL
# Non-custodial at T1_UK_RAL and T3_CO_Uniandes
# Autoapproved in all sites
# Priority is normal
self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed")
subInfoA = subscriptions[self.testDatasetA]
self.assertEqual(len(subInfoA), 3, "Dataset A was not subscribed to all sites")
for subInfo in subInfoA:
site = subInfo["node"]
self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription")
if site == "T1_UK_RAL_MSS" or site == "T3_CO_Uniandes":
self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "n",
"Wrong requestOnly for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"])
elif site == "T1_US_FNAL_MSS":
self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "n",
"Wrong requestOnly for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "y", "Wrong subscription type for dataset A at %s" % subInfo["node"])
else:
self.fail("Dataset A was subscribed to a wrong site %s" % site)
# Now check /BogusPrimary/CRUZET11-v1/RAW
# According to the spec, this is not custodial anywhere
# Non-custodial at T1_UK_RAL and T2_CH_CERN
# Request only at both sites and with high priority
self.assertTrue(self.testDatasetB in subscriptions, "Dataset B was not subscribed")
subInfoB = subscriptions[self.testDatasetB]
self.assertEqual(len(subInfoB), 2, "Dataset B was not subscribed to all sites")
for subInfo in subInfoB:
site = subInfo["node"]
self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription")
if site == "T1_UK_RAL_MSS" or site == "T2_CH_CERN":
self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset B at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "y",
"Wrong requestOnly for dataset B at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset B at %s" % subInfo["node"])
else:
self.fail("Dataset B was subscribed to a wrong site %s" % site)
myThread = threading.currentThread()
result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset_subscription where subscribed = 1")[
0].fetchall()
self.assertEqual(result[0][0], 5, "Not all datasets were marked as subscribed")
result = myThread.dbi.processData("SELECT site FROM dbsbuffer_dataset_subscription where subscribed = 0")[
0].fetchall()
self.assertEqual(result[0][0], "T1_IT_CNAF", "A non-valid CMS site was subscribed")
# Reset and run again and make sure that no duplicate subscriptions are created
myThread.dbi.processData("UPDATE dbsbuffer_dataset_subscription SET subscribed = 0")
subscriber.subscribeDatasets()
self.assertEqual(len(subscriptions[self.testDatasetA]), 3)
self.assertEqual(len(subscriptions[self.testDatasetB]), 2)
return
class PhEDExInjectorPollerTest(unittest.TestCase):
"""
_PhEDExInjectorPollerTest_
Unit tests for the PhEDExInjector. Create some database inside DBSBuffer
and then have the PhEDExInjector upload the data to PhEDEx. Pull the data
back down and verify that everything is complete.
"""
def setUp(self):
"""
_setUp_
Install the DBSBuffer schema into the database and connect to PhEDEx.
"""
self.phedexURL = "https://cmsweb.cern.ch/phedex/datasvc/json/test"
self.dbsURL = "http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet"
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase = True)
self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer"],
useDefault = False)
myThread = threading.currentThread()
daofactory = DAOFactory(package = "WMComponent.DBSBuffer.Database",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = daofactory(classname = "DBSBufferFiles.AddLocation")
locationAction.execute(siteName = "srm-cms.cern.ch")
locationAction.execute(siteName = "se.fnal.gov")
self.testFilesA = []
self.testFilesB = []
self.testDatasetA = "/%s/PromptReco-v1/RECO" % makeUUID()
self.testDatasetB = "/%s/CRUZET11-v1/RAW" % makeUUID()
self.phedex = PhEDEx({"endpoint": self.phedexURL}, "json")
return
def tearDown(self):
"""
_tearDown_
Delete the database.
"""
self.testInit.clearDatabase()
def stuffDatabase(self):
"""
_stuffDatabase_
Fill the dbsbuffer with some files and blocks. We'll insert a total
of 5 files spanning two blocks. There will be a total of two datasets
inserted into the datbase.
We'll inject files with the location set as an SE name as well as a
PhEDEx node name as well.
"""
myThread = threading.currentThread()
buffer3Factory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
insertWorkflow = buffer3Factory(classname = "InsertWorkflow")
insertWorkflow.execute("BogusRequest", "BogusTask",
0, 0, 0, 0)
checksums = {"adler32": "1234", "cksum": "5678"}
testFileA = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileA.setDatasetPath(self.testDatasetA)
testFileA.addRun(Run(2, *[45]))
testFileA.create()
testFileB = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileB.setDatasetPath(self.testDatasetA)
testFileB.addRun(Run(2, *[45]))
testFileB.create()
testFileC = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileC.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileC.setDatasetPath(self.testDatasetA)
testFileC.addRun(Run(2, *[45]))
testFileC.create()
self.testFilesA.append(testFileA)
self.testFilesA.append(testFileB)
self.testFilesA.append(testFileC)
testFileD = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileD.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileD.setDatasetPath(self.testDatasetB)
testFileD.addRun(Run(2, *[45]))
testFileD.create()
testFileE = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileE.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileE.setDatasetPath(self.testDatasetB)
testFileE.addRun(Run(2, *[45]))
testFileE.create()
self.testFilesB.append(testFileD)
self.testFilesB.append(testFileE)
uploadFactory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
datasetAction = uploadFactory(classname = "NewDataset")
createAction = uploadFactory(classname = "CreateBlocks")
datasetAction.execute(datasetPath = self.testDatasetA)
datasetAction.execute(datasetPath = self.testDatasetB)
self.blockAName = self.testDatasetA + "#" + makeUUID()
self.blockBName = self.testDatasetB + "#" + makeUUID()
newBlockA = DBSBufferBlock(name = self.blockAName,
location = "srm-cms.cern.ch",
datasetpath = None)
newBlockA.setDataset(self.testDatasetA, 'data', 'VALID')
newBlockA.status = 'Closed'
newBlockB = DBSBufferBlock(name = self.blockBName,
location = "srm-cms.cern.ch",
datasetpath = None)
newBlockB.setDataset(self.testDatasetB, 'data', 'VALID')
newBlockB.status = 'Closed'
createAction.execute(blocks = [newBlockA, newBlockB])
bufferFactory = DAOFactory(package = "WMComponent.DBSBuffer.Database",
logger = myThread.logger,
dbinterface = myThread.dbi)
setBlock = bufferFactory(classname = "DBSBufferFiles.SetBlock")
setBlock.execute(testFileA["lfn"], self.blockAName)
setBlock.execute(testFileB["lfn"], self.blockAName)
setBlock.execute(testFileC["lfn"], self.blockAName)
setBlock.execute(testFileD["lfn"], self.blockBName)
setBlock.execute(testFileE["lfn"], self.blockBName)
fileStatus = bufferFactory(classname = "DBSBufferFiles.SetStatus")
fileStatus.execute(testFileA["lfn"], "LOCAL")
fileStatus.execute(testFileB["lfn"], "LOCAL")
fileStatus.execute(testFileC["lfn"], "LOCAL")
fileStatus.execute(testFileD["lfn"], "LOCAL")
fileStatus.execute(testFileE["lfn"], "LOCAL")
associateWorkflow = buffer3Factory(classname = "DBSBufferFiles.AssociateWorkflowToFile")
associateWorkflow.execute(testFileA["lfn"], "BogusRequest", "BogusTask")
associateWorkflow.execute(testFileB["lfn"], "BogusRequest", "BogusTask")
associateWorkflow.execute(testFileC["lfn"], "BogusRequest", "BogusTask")
associateWorkflow.execute(testFileD["lfn"], "BogusRequest", "BogusTask")
associateWorkflow.execute(testFileE["lfn"], "BogusRequest", "BogusTask")
return
def createConfig(self):
"""
_createConfig_
Create a config for the PhEDExInjector with paths to the test DBS and
PhEDEx instances.
"""
config = self.testInit.getConfiguration()
config.component_("DBSInterface")
config.DBSInterface.globalDBSUrl = self.dbsURL
config.component_("PhEDExInjector")
config.PhEDExInjector.phedexurl = self.phedexURL
config.PhEDExInjector.subscribeMSS = True
config.PhEDExInjector.group = "Saturn"
config.PhEDExInjector.pollInterval = 30
config.PhEDExInjector.subscribeInterval = 60
return config
def retrieveReplicaInfoForBlock(self, blockName):
"""
_retrieveReplicaInfoForBlock_
Retrieve the replica information for a block. It takes several minutes
after a block is injected for the statistics to be calculated, so this
will block until that information is available.
"""
attempts = 0
while attempts < 15:
result = self.phedex.getReplicaInfoForFiles(block = blockName)
if "phedex" in result:
if "block" in result["phedex"]:
if len(result["phedex"]["block"]) != 0:
return result["phedex"]["block"][0]
attempts += 1
time.sleep(20)
logging.info("Could not retrieve replica info for block: %s" % blockName)
return None
@attr("integration")
def testPoller(self):
"""
_testPoller_
Stuff the database and have the poller upload files to PhEDEx. Retrieve
replica information for the uploaded blocks and verify that all files
have been injected.
"""
return
self.stuffDatabase()
poller = PhEDExInjectorPoller(self.createConfig())
poller.setup(parameters = None)
poller.algorithm(parameters = None)
replicaInfo = self.retrieveReplicaInfoForBlock(self.blockAName)
goldenLFNs = []
for file in self.testFilesA:
goldenLFNs.append(file["lfn"])
for replicaFile in replicaInfo["file"]:
assert replicaFile["name"] in goldenLFNs, \
"Error: Extra file in replica block: %s" % replicaFile["name"]
goldenLFNs.remove(replicaFile["name"])
assert len(goldenLFNs) == 0, \
"Error: Files missing from PhEDEx replica: %s" % goldenLFNs
replicaInfo = self.retrieveReplicaInfoForBlock(self.blockBName)
goldenLFNs = []
for file in self.testFilesB:
goldenLFNs.append(file["lfn"])
for replicaFile in replicaInfo["file"]:
assert replicaFile["name"] in goldenLFNs, \
"Error: Extra file in replica block: %s" % replicaFile["name"]
goldenLFNs.remove(replicaFile["name"])
assert len(goldenLFNs) == 0, \
"Error: Files missing from PhEDEx replica: %s" % goldenLFNs
myThread = threading.currentThread()
daofactory = DAOFactory(package = "WMComponent.DBSUpload.Database",
logger = myThread.logger,
dbinterface = myThread.dbi)
setBlock = daofactory(classname = "SetBlockStatus")
setBlock.execute(self.blockAName, locations = None,
open_status = "InGlobalDBS")
poller.algorithm(parameters = None)
replicaInfo = self.retrieveReplicaInfoForBlock(self.blockAName)
assert replicaInfo["is_open"] == "n", \
"Error: block should be closed."
replicaInfo = self.retrieveReplicaInfoForBlock(self.blockBName)
assert replicaInfo["is_open"] == "y", \
"Error: block should be open."
return
def test_CustodialSiteA(self):
"""
_CustodialSiteA_
Check the custodialSite stuff by DAO, since I don't have a cert
First make sure we properly handle having no custodialSite
"""
self.stuffDatabase()
myThread = threading.currentThread()
daofactory = DAOFactory(package = "WMComponent.PhEDExInjector.Database",
logger = myThread.logger,
dbinterface = myThread.dbi)
getUninjected = daofactory(classname = "GetUninjectedFiles")
uninjectedFiles = getUninjected.execute()
self.assertEqual(uninjectedFiles.keys(), ['srm-cms.cern.ch'])
return
class scaleTestFiller:
"""
_scaleTestFiller_
Initializes the DB and the DBSUploader
On __call__() it creates data and uploads it.
"""
def __init__(self):
"""
__init__
Init the DB
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase=True)
self.testInit.setSchema(customModules=["WMComponent.DBS3Buffer"],
useDefault=False)
self.configFile = EmulatorSetup.setupWMAgentConfig()
myThread = threading.currentThread()
self.bufferFactory = DAOFactory(
package="WMComponent.DBSBuffer.Database",
logger=myThread.logger,
dbinterface=myThread.dbi)
locationAction = self.bufferFactory(
classname="DBSBufferFiles.AddLocation")
locationAction.execute(siteName="se1.cern.ch")
locationAction.execute(siteName="se1.fnal.gov")
locationAction.execute(siteName="malpaquet")
config = self.getConfig()
self.dbsUploader = DBSUploadPoller(config=config)
return
def __call__(self):
"""
__call__
Generate some random data
"""
# Generate somewhere between one and a thousand files
name = "ThisIsATest_%s" % (makeUUID())
nFiles = random.randint(10, 2000)
name = name.replace('-', '_')
name = '%s-v0' % name
files = self.getFiles(name=name, nFiles=nFiles)
print "Inserting %i files for dataset %s" % (nFiles * 2, name)
try:
self.dbsUploader.algorithm()
except:
self.dbsUploader.close()
raise
# Repeat just to make sure
try:
self.dbsUploader.algorithm()
except:
self.dbsUploader.close()
raise
return
def getConfig(self):
"""
_getConfig_
This creates the actual config file used by the component
"""
config = self.testInit.getConfiguration()
self.testInit.generateWorkDir(config)
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = 'DBSUpload'
config.Agent.useHeartbeat = False
#Now the CoreDatabase information
#This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("DBS3Upload")
config.DBS3Upload.pollInterval = 10
config.DBS3Upload.logLevel = 'DEBUG'
config.DBS3Upload.DBSBlockMaxFiles = 500
config.DBS3Upload.DBSBlockMaxTime = 600
config.DBS3Upload.DBSBlockMaxSize = 999999999999
config.DBS3Upload.dbsUrl = 'http://cms-xen40.fnal.gov:8787/dbs/prod/global/DBSWriter'
config.DBS3Upload.namespace = 'WMComponent.DBS3Buffer.DBSUpload'
config.DBS3Upload.componentDir = os.path.join(os.getcwd(),
'Components')
config.DBS3Upload.nProcesses = 1
config.DBS3Upload.dbsWaitTime = 1
return config
def getFiles(self,
name,
tier='RECO',
nFiles=12,
site="malpaquet",
nLumis=1):
"""
Create some quick dummy test files
"""
files = []
for f in range(nFiles):
testFile = DBSBufferFile(
lfn='/data/store/random/random/RANDOM/test/0/%s-%s-%i.root' %
(name, site, f),
size=1024,
events=20,
checksums={'cksum': 1})
testFile.setAlgorithm(appName=name,
appVer="CMSSW_3_1_1",
appFam="RECO",
psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFile.setDatasetPath("/%s/%s/%s" % (name, name, tier))
lumis = []
for i in range(nLumis):
lumis.append((f * 100000) + i)
testFile.addRun(Run(1, *lumis))
testFile.setAcquisitionEra(name.split('-')[0])
testFile.setProcessingVer("0")
testFile.setGlobalTag("Weird")
testFile.create()
testFile.setLocation(site)
files.append(testFile)
count = 0
for f in files:
count += 1
testFileChild = DBSBufferFile(
lfn=
'/data/store/random/random/RANDOM/test/0/%s-%s-%i-child.root' %
(name, site, count),
size=1024,
events=10,
checksums={'cksum': 1})
testFileChild.setAlgorithm(appName=name,
appVer="CMSSW_3_1_1",
appFam="RECO",
psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFileChild.setDatasetPath("/%s/%s_2/RECO" % (name, name))
testFileChild.addRun(Run(1, *[45]))
testFileChild.create()
testFileChild.setLocation(site)
testFileChild.addParents([f['lfn']])
return files
class WorkflowManagerTest(unittest.TestCase):
"""
TestCase for TestWorkflowManager module
"""
_maxMessage = 10
def setUp(self):
"""
_setUp_
Setup the database and logging connection. Try to create all needed
WMBS tables.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = \
['WMCore.Agent.Database',
'WMComponent.WorkflowManager.Database',
'WMCore.ThreadPool',
'WMCore.MsgService',
'WMCore.WMBS'],
useDefault = False)
return
def tearDown(self):
"""
_tearDown_
Database deletion
"""
self.testInit.clearDatabase()
return
def getConfig(self):
"""
_getConfig_
Get defaults WorkflowManager parameters
"""
return self.testInit.getConfiguration(
os.path.join(WMCore.WMInit.getWMBASE(), \
'src/python/WMComponent/WorkflowManager/DefaultConfig.py'))
def testA(self):
"""
_testA_
Handle AddWorkflowToManage events
"""
myThread = threading.currentThread()
config = self.getConfig()
testWorkflowManager = WorkflowManager(config)
testWorkflowManager.prepareToStart()
for i in xrange(0, WorkflowManagerTest._maxMessage):
workflow = Workflow(spec = "testSpec.xml", owner = "riahi", \
name = "testWorkflow" + str(i), task = "testTask")
workflow.create()
for j in xrange(0, 3):
workflowManagerdict = {'payload':{'WorkflowId' : workflow.id \
, 'FilesetMatch': 'FILESET_' + str(j) ,'SplitAlgo':'NO SPLITALGO', 'Type':'NO TYPE' }}
testWorkflowManager.handleMessage( \
type = 'AddWorkflowToManage' , payload = workflowManagerdict )
time.sleep(30)
myThread.workerThreadManager.terminateWorkers()
while threading.activeCount() > 1:
print('Currently: '+str(threading.activeCount())+\
' Threads. Wait until all our threads have finished')
time.sleep(1)
class HarvestTest(unittest.TestCase):
"""
_HarvestTest_
Test for EndOfRun job splitter
"""
def setUp(self):
"""
_setUp_
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=["WMCore.WMBS"])
self.splitterFactory = SplitterFactory(package="WMCore.JobSplitting")
myThread = threading.currentThread()
self.myThread = myThread
daoFactory = DAOFactory(package="WMCore.WMBS",
logger=logging,
dbinterface=myThread.dbi)
self.WMBSFactory = daoFactory
config = self.getConfig()
self.changer = ChangeState(config)
myResourceControl = ResourceControl()
myResourceControl.insertSite("T1_US_FNAL", 10, 20, "T1_US_FNAL_Disk", "T1_US_FNAL")
myResourceControl.insertSite("T1_US_FNAL", 10, 20, "T3_US_FNALLPC", "T1_US_FNAL")
myResourceControl.insertSite("T2_CH_CERN", 10, 20, "T2_CH_CERN", "T2_CH_CERN")
self.fileset1 = Fileset(name="TestFileset1")
for fileNum in range(11):
newFile = File("/some/file/name%d" % fileNum, size=1000, events=100)
newFile.addRun(Run(1, *[1]))
newFile.setLocation('T1_US_FNAL_Disk')
self.fileset1.addFile(newFile)
self.fileset1.create()
workflow1 = Workflow(spec="spec.xml", owner="hufnagel", name="TestWorkflow1", task="Test")
workflow1.create()
self.subscription1 = Subscription(fileset=self.fileset1,
workflow=workflow1,
split_algo="Harvest",
type="Harvesting")
self.subscription1.create()
self.configFile = EmulatorSetup.setupWMAgentConfig()
return
def tearDown(self):
"""
_tearDown_
"""
self.testInit.clearDatabase()
EmulatorSetup.deleteConfig(self.configFile)
return
def getConfig(self):
"""
_getConfig_
"""
config = self.testInit.getConfiguration()
self.testInit.generateWorkDir(config)
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL', None)
config.JobStateMachine.couchDBName = 'wmagent_jobdump'
return config
def finishJobs(self, jobGroups, subscription=None):
"""
_finishJobs_
"""
if not subscription:
subscription = self.subscription1
for f in subscription.acquiredFiles():
subscription.completeFiles(f)
for jobGroup in jobGroups:
self.changer.propagate(jobGroup.jobs, 'executing', 'created')
self.changer.propagate(jobGroup.jobs, 'complete', 'executing')
self.changer.propagate(jobGroup.jobs, 'success', 'complete')
self.changer.propagate(jobGroup.jobs, 'cleanout', 'success')
return
def testHarvestEndOfRunTrigger(self):
"""
_testDQMHarvestEndOfRunTrigger_
Make sure that the basic splitting algo works, which is only, ExpressMerge is ALL done, fire a job against that fileset
"""
self.assertEqual(self.fileset1.open, True, "Fileset is closed. Shouldn't")
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)
jobGroups = jobFactory()
self.assertEqual(len(jobGroups), 0,
"We got 1 or more jobGroups with an open fileset and no periodic configuration")
self.fileset1.markOpen(False)
self.assertEqual(self.fileset1.open, False, "Fileset is opened, why?")
# We should also check if there are aqcuired files, if there are, there are jobs,
# we don't want to fire another jobs while previous are running (output is integrating whatever input)
# TODO : The above one we can do when all is done. Not priority
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)
jobGroups = jobFactory()
self.assertEqual(len(jobGroups), 1,
"Harvest jobsplitter didn't create a single jobGroup after the fileset was closed")
return
def testPeriodicTrigger(self):
"""
_testPeriodicTrigger_
"""
self.assertEqual(self.fileset1.open, True, "Fileset is not open, not testing periodic here")
# Test timeout (5s for this first test)
# there should be no acquired files, if there are, shouldn't be a job
# self.subscription1.acquireFiles(self.subscription1.availableFiles().pop())
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval=3)
self.assertEqual(len(jobGroups), 1, "Didn't created the first periodic job when there were acquired files")
# For the whole thing to work, faking the first job finishing, and putting the files as complete
self.finishJobs(jobGroups)
# Adding more of files, so we have new stuff to process
for fileNum in range(12, 24):
newFile = File("/some/file/name%d" % fileNum, size=1000, events=100)
newFile.addRun(Run(1, *[1]))
newFile.setLocation('T1_US_FNAL_Disk')
self.fileset1.addFile(newFile)
self.fileset1.commit()
# Testing that it doesn't create a job unless the delay is past
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval=2)
self.assertEqual(len(jobGroups), 0,
"Created one or more job, when there were non-acquired file and the period is not passed by")
time.sleep(2)
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval=2)
self.assertEqual(len(jobGroups), 1,
"Didn't created one or more job, and there weren't and the period is passed by")
# Finishing out previous jobs
self.finishJobs(jobGroups)
# Adding more of files, so we have new stuff to process
for fileNum in range(26, 36):
newFile = File("/some/file/name%d" % fileNum, size=1000, events=100)
newFile.addRun(Run(1, *[1]))
newFile.setLocation('T1_US_FNAL_Disk')
self.fileset1.addFile(newFile)
self.fileset1.commit()
# Trying to create another job just afterwards, it shouldn't, because it should respect the configured delay
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval=2)
self.assertEqual(len(jobGroups), 0, "Created one or more job, there are new files, but the delay is not past")
time.sleep(2)
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval=2)
self.assertEqual(len(jobGroups), 1, "Didn't created one or more job, there are new files and the delay is past")
# Last check is whether the job gets all the files or not
numFilesJob = jobGroups[0].jobs[0].getFiles()
numFilesFileset = self.fileset1.getFiles()
self.assertEqual(numFilesJob, numFilesFileset, "Job didn't got all the files")
# Finishing out previous jobs
self.finishJobs(jobGroups)
# Adding files for the first location
for fileNum in range(38, 48):
newFile = File("/some/file/name%d" % fileNum, size=1000, events=100)
newFile.addRun(Run(1, *[1]))
newFile.setLocation('T1_US_FNAL_Disk')
self.fileset1.addFile(newFile)
self.fileset1.commit()
# Then another location
for fileNum in range(50, 56):
newFile = File("/some/file/name%d" % fileNum, size=1000, events=100)
newFile.addRun(Run(1, *[1]))
newFile.setLocation('T2_CH_CERN')
self.fileset1.addFile(newFile)
self.fileset1.commit()
# We should have jobs in both locations
time.sleep(2)
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval=2)
self.assertEqual(len(jobGroups[0].getJobs()), 2, "We didn't get 2 jobs for 2 locations")
firstJobLocation = jobGroups[0].getJobs()[0].getFileLocations()[0]
secondJobLocation = jobGroups[0].getJobs()[1].getFileLocations()[0]
self.assertEqual(firstJobLocation, 'T2_CH_CERN')
self.assertEqual(secondJobLocation, 'T1_US_FNAL')
self.finishJobs(jobGroups)
for fileNum in range(60, 65):
newFile = File("/some/file/name%d" % fileNum, size=1000, events=100)
newFile.addRun(Run(2, *[2]))
newFile.setLocation('T2_CH_CERN')
self.fileset1.addFile(newFile)
self.fileset1.commit()
for fileNum in range(70, 75):
newFile = File("/some/file/name%d" % fileNum, size=1000, events=100)
newFile.addRun(Run(3, *[3]))
newFile.setLocation('T2_CH_CERN')
self.fileset1.addFile(newFile)
self.fileset1.commit()
time.sleep(2)
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1)
jobGroups = jobFactory(periodic_harvest_interval=2)
# This is one of the most "complicated" tests so worth to comment, 4 jobs should be created
# 1 - all previous files from SomeSE and run = 1 (a lot, like ~45)
# 2 - Few files from SomeSE3, Run = 1
# 3 - Few files from SomeSE3, Run = 2
# 4 - Few files from SomeSE3, Run = 3
self.assertEqual(len(jobGroups[0].getJobs()), 4, "We didn't get 4 jobs for adding 2 different runs to SomeSE3")
return
def testMultipleRunHarvesting(self):
"""
_testMultipleRunHarvesting_
Add some files with multiple runs in each, make sure the jobs
are created by location and run. Verify each job mask afterwards.
Note that in this test run are splitted between sites,
in real life that MUST NOT happen we still don't support that.
"""
multipleFilesFileset = Fileset(name="TestFileset")
newFile = File("/some/file/test1", size=1000, events=100)
newFile.addRun(Run(1, *[1, 3, 4, 5, 6, 7]))
newFile.addRun(Run(2, *[1, 2, 4, 5, 6, 7]))
newFile.setLocation('T1_US_FNAL_Disk')
multipleFilesFileset.addFile(newFile)
newFile = File("/some/file/test2", size=1000, events=100)
newFile.addRun(Run(1, *[2, 8]))
newFile.addRun(Run(2, *[3, 8]))
newFile.setLocation('T2_CH_CERN')
multipleFilesFileset.addFile(newFile)
multipleFilesFileset.create()
harvestingWorkflow = Workflow(spec="spec.xml",
owner="hufnagel",
name="TestWorkflow",
task="Test")
harvestingWorkflow.create()
harvestSub = Subscription(fileset=multipleFilesFileset,
workflow=harvestingWorkflow,
split_algo="Harvest",
type="Harvesting")
harvestSub.create()
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub)
jobGroups = jobFactory(periodic_harvest_interval=2)
self.assertEqual(len(jobGroups), 1, "A single job group was not created")
self.assertEqual(len(jobGroups[0].getJobs()), 4,
"Four jobs were not created")
for job in jobGroups[0].getJobs():
runs = job['mask'].getRunAndLumis()
self.assertEqual(len(runs), 1, "Job has more than one run configured")
ll = LumiList(compactList={1: [[1, 1], [3, 7], [2, 2], [8, 8]],
2: [[1, 2], [4, 7], [3, 3], [8, 8]]})
run = runs.keys()[0]
for lumiPair in runs[run]:
for lumi in range(lumiPair[0], lumiPair[1] + 1):
self.assertTrue((str(run), lumi) in ll, "All of %s not in %s" % (lumiPair, ll))
self.finishJobs(jobGroups, harvestSub)
newFile = File("/some/file/test3", size=1000, events=100)
newFile.addRun(Run(1, *range(9, 15)))
newFile.setLocation('T2_CH_CERN')
multipleFilesFileset.addFile(newFile)
multipleFilesFileset.commit()
time.sleep(2)
jobGroups = jobFactory(periodic_harvest_interval=2)
self.assertEqual(len(jobGroups), 1, "A single job group was not created")
self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created")
for job in jobGroups[0].getJobs():
runs = job['mask'].getRunAndLumis()
self.assertEqual(len(runs), 1, "Job has more than one run configured")
ll = LumiList(compactList={1: [[1, 1], [3, 7], [2, 2], [8, 8], [9, 14]],
2: [[1, 2], [4, 7], [3, 3], [8, 8]]})
run = runs.keys()[0]
for lumiPair in runs[run]:
for lumi in range(lumiPair[0], lumiPair[1] + 1):
self.assertTrue((run, lumi) in ll, "All of %s not in %s" % (lumiPair, ll))
harvestingWorkflowSib = Workflow(spec="spec.xml",
owner="hufnagel",
name="TestWorkflowSib",
task="TestSib")
harvestingWorkflowSib.create()
harvestSubSib = Subscription(fileset=multipleFilesFileset,
workflow=harvestingWorkflowSib,
split_algo="Harvest",
type="Harvesting")
harvestSubSib.create()
jobFactorySib = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSubSib)
multipleFilesFileset.markOpen(False)
jobGroups = jobFactorySib(periodic_harvest_sibling=True)
self.assertEqual(len(jobGroups), 0, "A single job group was created")
self.finishJobs(jobGroups, harvestSub)
jobGroups = jobFactorySib(periodic_harvest_sibling=True)
self.assertEqual(len(jobGroups), 1, "A single job group was not created")
self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created")
for job in jobGroups[0].getJobs():
runs = job['mask'].getRunAndLumis()
self.assertEqual(len(runs), 1, "Job has more than one run configured")
ll = LumiList(compactList={1: [[1, 1], [3, 7], [2, 2], [8, 8], [9, 14]],
2: [[1, 2], [4, 7], [3, 3], [8, 8]]})
run = runs.keys()[0]
for lumiPair in runs[run]:
for lumi in range(lumiPair[0], lumiPair[1] + 1):
self.assertTrue((run, lumi) in ll, "All of %s not in %s" % (lumiPair, ll))
def testMultiRunHarvesting(self):
"""
_testMultiRunHarvesting_
Provided a fileset with a couple of files and different runs, create a
single job for all the runs at a specific location, which also adds a
baggage to the job (True) which is later on looked up by SetupCMSSWPSet.
"""
multipleFilesFileset = createCommonFileset()
self.assertEqual(multipleFilesFileset.open, True)
harvestingWorkflow = Workflow(spec="spec.xml",
owner="amaltaro",
name="TestWorkflow",
task="Test")
harvestingWorkflow.create()
harvestSub = Subscription(fileset=multipleFilesFileset,
workflow=harvestingWorkflow,
split_algo="Harvest",
type="Harvesting")
harvestSub.create()
multipleFilesFileset.markOpen(False)
self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed")
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub)
jobGroups = jobFactory(dqmHarvestUnit="multiRun")
self.assertEqual(len(jobGroups), 1)
for jobGroup in jobGroups:
self.assertEqual(len(jobGroup.jobs), 1)
for job in jobGroup.jobs:
baggage = job.getBaggage()
self.assertTrue(getattr(baggage, "multiRun", False), "It's supposed to be a multiRun job")
self.assertEqual(getattr(baggage, "runLimits", ""), "-1-6")
def testByRunHarvesting(self):
"""
_testByRunHarvesting_
Provided a fileset with a couple of files and 4 different runs, create
one single job per run and location.
The multiRun baggage should be false in this case.
"""
multipleFilesFileset = createCommonFileset()
self.assertEqual(multipleFilesFileset.open, True, "Fileset should be open!")
harvestingWorkflow = Workflow(spec="spec.xml",
owner="amaltaro",
name="TestWorkflow",
task="Test")
harvestingWorkflow.create()
harvestSub = Subscription(fileset=multipleFilesFileset,
workflow=harvestingWorkflow,
split_algo="Harvest",
type="Harvesting")
harvestSub.create()
multipleFilesFileset.markOpen(False)
self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed")
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub)
jobGroups = jobFactory()
self.assertEqual(len(jobGroups), 1, "Should have created 1 job group")
for jobGroup in jobGroups:
self.assertEqual(len(jobGroup.jobs), 6, "Should have created 6 jobs")
for job in jobGroup.jobs:
baggage = job.getBaggage()
self.assertFalse(getattr(baggage, "multiRun", False), "It's supposed to be a byRun job")
def testByRunAndRunWhitelist(self):
"""
_testByRunAndRunWhitelist_
Create harvesting jobs by run for the runs provided in the RunWhitelist
"""
multipleFilesFileset = createCommonFileset()
self.assertEqual(multipleFilesFileset.open, True)
harvestingWorkflow = Workflow(spec="spec.xml", owner="amaltaro",
name="TestWorkflow", task="Test")
harvestingWorkflow.create()
harvestSub = Subscription(fileset=multipleFilesFileset, workflow=harvestingWorkflow,
split_algo="Harvest", type="Harvesting")
harvestSub.create()
multipleFilesFileset.markOpen(False)
self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed")
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub)
jobGroups = jobFactory(runWhitelist=[1, 3])
self.assertEqual(len(jobGroups), 1, "One jobgroup per location")
for jobGroup in jobGroups:
self.assertEqual(len(jobGroup.jobs), 2)
def testByRunAndRunBlacklist(self):
"""
_testByRunAndRunWhitelist_
Create harvesting jobs by run for the runs provided in the RunWhitelist
"""
multipleFilesFileset = createCommonFileset()
self.assertEqual(multipleFilesFileset.open, True)
harvestingWorkflow = Workflow(spec="spec.xml", owner="amaltaro",
name="TestWorkflow", task="Test")
harvestingWorkflow.create()
harvestSub = Subscription(fileset=multipleFilesFileset, workflow=harvestingWorkflow,
split_algo="Harvest", type="Harvesting")
harvestSub.create()
multipleFilesFileset.markOpen(False)
self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed")
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub)
jobGroups = jobFactory(runWhitelist=[1, 2, 3, 4, 5], runBlacklist=[1, 3])
self.assertEqual(len(jobGroups), 1, "One jobgroup per location")
for jobGroup in jobGroups:
self.assertEqual(len(jobGroup.jobs), 3)
class DBSUploadTest(unittest.TestCase):
"""
TestCase for DBSUpload module
"""
def setUp(self):
"""
_setUp_
setUp function for unittest
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer"],
useDefault = False)
self.testDir = self.testInit.generateWorkDir(deleteOnDestruction = False)
self.configFile = EmulatorSetup.setupWMAgentConfig()
myThread = threading.currentThread()
self.bufferFactory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = self.bufferFactory(classname = "DBSBufferFiles.AddLocation")
locationAction.execute(siteName = "se1.cern.ch")
locationAction.execute(siteName = "se1.fnal.gov")
locationAction.execute(siteName = "malpaquet")
self.dbsUrl = "https://*****:*****@attr("integration")
def testBasicUpload(self):
"""
_testBasicUpload_
Verify that we can successfully upload to DBS3. Also verify that the
uploader correctly handles files parentage when uploading.
"""
self.dbsApi = DbsApi(url = self.dbsUrl)
config = self.getConfig()
dbsUploader = DBSUploadPoller(config = config)
# First test verifies that uploader will poll and then not do anything
# as the database is empty.
dbsUploader.algorithm()
acqEra = "Summer%s" % (int(time.time()))
parentFiles = self.createParentFiles(acqEra)
# The algorithm needs to be run twice. On the first iteration it will
# create all the blocks and upload one. On the second iteration it will
# timeout and upload the second block.
dbsUploader.algorithm()
time.sleep(5)
dbsUploader.algorithm()
time.sleep(5)
# Verify the files made it into DBS3.
self.verifyData(parentFiles[0]["datasetPath"], parentFiles)
# Inject some more parent files and some child files into DBSBuffer.
# Run the uploader twice, only the parent files should be added to DBS3.
(moreParentFiles, childFiles) = \
self.createFilesWithChildren(parentFiles, acqEra)
dbsUploader.algorithm()
time.sleep(5)
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(parentFiles[0]["datasetPath"],
parentFiles + moreParentFiles)
# Run the uploader another two times to upload the child files. Verify
# that the child files were uploaded.
dbsUploader.algorithm()
time.sleep(5)
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(childFiles[0]["datasetPath"], childFiles)
return
@attr("integration")
def testDualUpload(self):
"""
_testDualUpload_
Verify that the dual upload mode works correctly.
"""
self.dbsApi = DbsApi(url = self.dbsUrl)
config = self.getConfig()
dbsUploader = DBSUploadPoller(config = config)
dbsUtil = DBSBufferUtil()
# First test verifies that uploader will poll and then not do anything
# as the database is empty.
dbsUploader.algorithm()
acqEra = "Summer%s" % (int(time.time()))
parentFiles = self.createParentFiles(acqEra)
(moreParentFiles, childFiles) = \
self.createFilesWithChildren(parentFiles, acqEra)
allFiles = parentFiles + moreParentFiles
allBlocks = []
for i in range(4):
DBSBufferDataset(parentFiles[0]["datasetPath"]).create()
blockName = parentFiles[0]["datasetPath"] + "#" + makeUUID()
dbsBlock = DBSBufferBlock(blockName,
location = "malpaquet",
datasetpath = None)
dbsBlock.status = "Open"
dbsBlock.setDataset(parentFiles[0]["datasetPath"], 'data', 'VALID')
dbsUtil.createBlocks([dbsBlock])
for file in allFiles[i * 5 : (i * 5) + 5]:
dbsBlock.addFile(file, 'data', 'VALID')
dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]})
if i < 2:
dbsBlock.status = "InDBS"
dbsUtil.updateBlocks([dbsBlock])
dbsUtil.updateFileStatus([dbsBlock], "InDBS")
allBlocks.append(dbsBlock)
DBSBufferDataset(childFiles[0]["datasetPath"]).create()
blockName = childFiles[0]["datasetPath"] + "#" + makeUUID()
dbsBlock = DBSBufferBlock(blockName,
location = "malpaquet",
datasetpath = None)
dbsBlock.status = "InDBS"
dbsBlock.setDataset(childFiles[0]["datasetPath"], 'data', 'VALID')
dbsUtil.createBlocks([dbsBlock])
for file in childFiles:
dbsBlock.addFile(file, 'data', 'VALID')
dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]})
dbsUtil.updateFileStatus([dbsBlock], "InDBS")
dbsUploader.algorithm()
time.sleep(5)
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(parentFiles[0]["datasetPath"], parentFiles)
# Change the status of the rest of the parent blocks so we can upload
# them and the children.
for dbsBlock in allBlocks:
dbsBlock.status = "InDBS"
dbsUtil.updateBlocks([dbsBlock])
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(parentFiles[0]["datasetPath"], parentFiles + moreParentFiles)
# Run the uploader one more time to upload the children.
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(childFiles[0]["datasetPath"], childFiles)
return
def testCloseSettingsPerWorkflow(self):
"""
_testCloseSettingsPerWorkflow_
Test the block closing mechanics in the DBS3 uploader,
this uses a fake dbs api to avoid reliance on external services.
"""
# Signal trapExit that we are a friend
os.environ["DONT_TRAP_EXIT"] = "True"
try:
# Monkey patch the imports of DbsApi
from WMComponent.DBS3Buffer import DBSUploadPoller as MockDBSUploadPoller
MockDBSUploadPoller.DbsApi = MockDbsApi
# Set the poller and the dbsUtil for verification
myThread = threading.currentThread()
(_, dbsFilePath) = mkstemp(dir = self.testDir)
self.dbsUrl = dbsFilePath
config = self.getConfig()
dbsUploader = MockDBSUploadPoller.DBSUploadPoller(config = config)
dbsUtil = DBSBufferUtil()
# First test is event based limits and timeout with no new files.
# Set the files and workflow
acqEra = "TropicalSeason%s" % (int(time.time()))
workflowName = 'TestWorkload%s' % (int(time.time()))
taskPath = '/%s/TestProcessing' % workflowName
self.injectWorkflow(workflowName, taskPath,
MaxWaitTime = 2, MaxFiles = 100,
MaxEvents = 150)
self.createParentFiles(acqEra, nFiles = 20,
workflowName = workflowName,
taskPath = taskPath)
# The algorithm needs to be run twice. On the first iteration it will
# create all the blocks and upload one with less than 150 events.
# On the second iteration the second block is uploaded.
dbsUploader.algorithm()
dbsUploader.checkBlocks()
openBlocks = dbsUtil.findOpenBlocks()
self.assertEqual(len(openBlocks), 1)
globalFiles = myThread.dbi.processData("SELECT id FROM dbsbuffer_file WHERE status = 'InDBS'")[0].fetchall()
notUploadedFiles = myThread.dbi.processData("SELECT id FROM dbsbuffer_file WHERE status = 'NOTUPLOADED'")[0].fetchall()
self.assertEqual(len(globalFiles), 14)
self.assertEqual(len(notUploadedFiles), 6)
# Check the fake DBS for data
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 2)
for block in fakeDBSInfo:
self.assertTrue('block_events' not in block['block'])
self.assertEqual(block['block']['file_count'], 7)
self.assertEqual(block['block']['open_for_writing'], 0)
self.assertTrue('close_settings' not in block)
time.sleep(3)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
openBlocks = dbsUtil.findOpenBlocks()
self.assertEqual(len(openBlocks), 0)
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 3)
for block in fakeDBSInfo:
if block['block']['file_count'] != 6:
self.assertEqual(block['block']['file_count'], 7)
self.assertTrue('block_events' not in block['block'])
self.assertEqual(block['block']['open_for_writing'], 0)
self.assertTrue('close_settings' not in block)
# Now check the limit by size and timeout with new files
acqEra = "TropicalSeason%s" % (int(time.time()))
workflowName = 'TestWorkload%s' % (int(time.time()))
taskPath = '/%s/TestProcessing' % workflowName
self.injectWorkflow(workflowName, taskPath,
MaxWaitTime = 2, MaxFiles = 5,
MaxEvents = 200000000)
self.createParentFiles(acqEra, nFiles = 16,
workflowName = workflowName,
taskPath = taskPath)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
openBlocks = dbsUtil.findOpenBlocks()
self.assertEqual(len(openBlocks), 1)
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 6)
for block in fakeDBSInfo:
if acqEra in block['block']['block_name']:
self.assertEqual(block['block']['file_count'], 5)
self.assertTrue('block_events' not in block['block'])
self.assertTrue('close_settings' not in block)
self.assertEqual(block['block']['open_for_writing'], 0)
# Put more files, they will go into the same block and then it will be closed
# after timeout
time.sleep(3)
self.createParentFiles(acqEra, nFiles = 3,
workflowName = workflowName,
taskPath = taskPath)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
openBlocks = dbsUtil.findOpenBlocks()
self.assertEqual(len(openBlocks), 0)
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 7)
for block in fakeDBSInfo:
if acqEra in block['block']['block_name']:
if block['block']['file_count'] < 5:
self.assertEqual(block['block']['file_count'], 4)
else:
self.assertEqual(block['block']['file_count'], 5)
self.assertTrue('block_events' not in block['block'])
self.assertEqual(block['block']['open_for_writing'], 0)
self.assertTrue('close_settings' not in block)
# Finally test size limits
acqEra = "TropicalSeason%s" % (int(time.time()))
workflowName = 'TestWorkload%s' % (int(time.time()))
taskPath = '/%s/TestProcessing' % workflowName
self.injectWorkflow(workflowName, taskPath,
MaxWaitTime = 1, MaxFiles = 500,
MaxEvents = 200000000, MaxSize = 2048)
self.createParentFiles(acqEra, nFiles = 7,
workflowName = workflowName,
taskPath = taskPath)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
time.sleep(2)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
self.assertEqual(len(openBlocks), 0)
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 11)
for block in fakeDBSInfo:
if acqEra in block['block']['block_name']:
if block['block']['file_count'] != 1:
self.assertEqual(block['block']['block_size'], 2048)
self.assertEqual(block['block']['file_count'], 2)
self.assertTrue('block_events' not in block['block'])
self.assertEqual(block['block']['open_for_writing'], 0)
self.assertTrue('close_settings' not in block)
except:
self.fail("We failed at some point in the test")
finally:
# We don't trust anyone else with _exit
del os.environ["DONT_TRAP_EXIT"]
return
class PhEDExInjectorSubscriberTest(unittest.TestCase):
"""
_PhEDExInjectorSubscriberTest_
Create some database inside DBSBuffer, run the subscriber algorithm
using a PhEDEx emulator and verify that it works both in unsafe and safe mode.
For unsafe mode there a WMBS database is also created
"""
def setUp(self):
"""
_setUp_
Install the DBSBuffer schema into the database and connect to PhEDEx.
"""
self.phedexURL = "https://bogus.cern.ch/bogus"
self.dbsURL = "https://bogus.cern.ch/bogus"
EmulatorHelper.setEmulators(phedex=True, dbs=True, siteDB=True)
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=["WMComponent.DBS3Buffer", "WMCore.WMBS"], useDefault=False)
self.testFilesA = []
self.testFilesB = []
self.testDatasetA = "/BogusPrimary/Run2012Z-PromptReco-v1/RECO"
self.testDatasetB = "/BogusPrimary/CRUZET11-v1/RAW"
return
def tearDown(self):
"""
_tearDown_
Delete the database.
"""
self.testInit.clearDatabase()
EmulatorHelper.resetEmulators()
def createConfig(self):
"""
_createConfig_
Create a config for the PhEDExInjector, paths to DBS and PhEDEx are dummies because
we are using Emulators
"""
config = self.testInit.getConfiguration()
config.component_("DBSInterface")
config.DBSInterface.globalDBSUrl = self.dbsURL
config.component_("PhEDExInjector")
config.PhEDExInjector.phedexurl = self.phedexURL
config.PhEDExInjector.subscribeDatasets = True
config.PhEDExInjector.group = "Saturn"
config.PhEDExInjector.pollInterval = 30
config.PhEDExInjector.subscribeInterval = 60
return config
def stuffDatabase(self):
"""
_stuffDatabase_
Fill the dbsbuffer with some files and blocks. We'll insert a total
of 5 files spanning two blocks. There will be a total of two datasets
inserted into the database.
All files will be already in GLOBAL and in_phedex
"""
myThread = threading.currentThread()
buffer3Factory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi)
insertWorkflow = buffer3Factory(classname="InsertWorkflow")
insertWorkflow.execute("BogusRequestA", "BogusTask", 0, 0, 0, 0)
insertWorkflow.execute("BogusRequestB", "BogusTask", 0, 0, 0, 0)
checksums = {"adler32": "1234", "cksum": "5678"}
testFileA = DBSBufferFile(
lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"])
)
testFileA.setAlgorithm(
appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH"
)
testFileA.setDatasetPath(self.testDatasetA)
testFileA.addRun(Run(2, *[45]))
testFileA.create()
testFileB = DBSBufferFile(
lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"])
)
testFileB.setAlgorithm(
appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH"
)
testFileB.setDatasetPath(self.testDatasetA)
testFileB.addRun(Run(2, *[45]))
testFileB.create()
testFileC = DBSBufferFile(
lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"])
)
testFileC.setAlgorithm(
appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH"
)
testFileC.setDatasetPath(self.testDatasetA)
testFileC.addRun(Run(2, *[45]))
testFileC.create()
self.testFilesA.append(testFileA)
self.testFilesA.append(testFileB)
self.testFilesA.append(testFileC)
testFileD = DBSBufferFile(
lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"])
)
testFileD.setAlgorithm(
appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH"
)
testFileD.setDatasetPath(self.testDatasetB)
testFileD.addRun(Run(2, *[45]))
testFileD.create()
testFileE = DBSBufferFile(
lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"])
)
testFileE.setAlgorithm(
appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH"
)
testFileE.setDatasetPath(self.testDatasetB)
testFileE.addRun(Run(2, *[45]))
testFileE.create()
self.testFilesB.append(testFileD)
self.testFilesB.append(testFileE)
uploadFactory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi)
datasetAction = uploadFactory(classname="NewDataset")
createAction = uploadFactory(classname="CreateBlocks")
datasetAction.execute(datasetPath=self.testDatasetA)
datasetAction.execute(datasetPath=self.testDatasetB)
self.blockAName = self.testDatasetA + "#" + makeUUID()
self.blockBName = self.testDatasetB + "#" + makeUUID()
newBlockA = DBSBufferBlock(name=self.blockAName, location="srm-cms.cern.ch", datasetpath=None)
newBlockA.setDataset(self.testDatasetA, "data", "VALID")
newBlockA.status = "Closed"
newBlockB = DBSBufferBlock(name=self.blockBName, location="srm-cms.cern.ch", datasetpath=None)
newBlockB.setDataset(self.testDatasetB, "data", "VALID")
newBlockB.status = "Closed"
createAction.execute(blocks=[newBlockA, newBlockB])
bufferFactory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi)
setBlock = bufferFactory(classname="DBSBufferFiles.SetBlock")
setBlock.execute(testFileA["lfn"], self.blockAName)
setBlock.execute(testFileB["lfn"], self.blockAName)
setBlock.execute(testFileC["lfn"], self.blockAName)
setBlock.execute(testFileD["lfn"], self.blockBName)
setBlock.execute(testFileE["lfn"], self.blockBName)
fileStatus = bufferFactory(classname="DBSBufferFiles.SetStatus")
fileStatus.execute(testFileA["lfn"], "GLOBAL")
fileStatus.execute(testFileB["lfn"], "GLOBAL")
fileStatus.execute(testFileC["lfn"], "GLOBAL")
fileStatus.execute(testFileD["lfn"], "GLOBAL")
fileStatus.execute(testFileE["lfn"], "GLOBAL")
phedexStatus = bufferFactory(classname="DBSBufferFiles.SetPhEDExStatus")
phedexStatus.execute(testFileA["lfn"], 1)
phedexStatus.execute(testFileB["lfn"], 1)
phedexStatus.execute(testFileC["lfn"], 1)
phedexStatus.execute(testFileD["lfn"], 1)
phedexStatus.execute(testFileE["lfn"], 1)
associateWorkflow = buffer3Factory(classname="DBSBufferFiles.AssociateWorkflowToFile")
associateWorkflow.execute(testFileA["lfn"], "BogusRequestA", "BogusTask")
associateWorkflow.execute(testFileB["lfn"], "BogusRequestA", "BogusTask")
associateWorkflow.execute(testFileC["lfn"], "BogusRequestA", "BogusTask")
associateWorkflow.execute(testFileD["lfn"], "BogusRequestB", "BogusTask")
associateWorkflow.execute(testFileE["lfn"], "BogusRequestB", "BogusTask")
# Make the desired subscriptions
insertSubAction = buffer3Factory(classname="NewSubscription")
datasetA = DBSBufferDataset(path=self.testDatasetA)
datasetB = DBSBufferDataset(path=self.testDatasetB)
workload = WMWorkloadHelper()
workload.load(os.path.join(getTestBase(), "WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl"))
insertSubAction.execute(datasetA.exists(), workload.getSubscriptionInformation()[self.testDatasetA])
insertSubAction.execute(datasetB.exists(), workload.getSubscriptionInformation()[self.testDatasetB])
return
def testNormalModeSubscriptions(self):
"""
_testNormalModeSubscriptions_
Tests that we can make custodial/non-custodial subscriptions on
normal operation mode, this time we don't need WMBS for anything.
All is subscribed in one go.
Check that the requests are correct.
"""
self.stuffDatabase()
config = self.createConfig()
subscriber = PhEDExInjectorPoller(config)
subscriber.setup({})
subscriber.subscribeDatasets()
phedexInstance = subscriber.phedex
subscriptions = phedexInstance.subRequests
# Let's check /BogusPrimary/Run2012Z-PromptReco-v1/RECO
# According to the spec, this should be custodial at T1_US_FNAL
# Non-custodial at T1_UK_RAL and T3_CO_Uniandes
# Autoapproved in all sites
# Priority is normal
self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed")
subInfoA = subscriptions[self.testDatasetA]
self.assertEqual(len(subInfoA), 3, "Dataset A was not subscribed to all sites")
for subInfo in subInfoA:
site = subInfo["node"]
self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription")
if site == "T1_UK_RAL_MSS" or site == "T3_CO_Uniandes":
self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(
subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]
)
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"])
elif site == "T1_US_FNAL_MSS":
self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(
subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]
)
self.assertEqual(subInfo["move"], "y", "Wrong subscription type for dataset A at %s" % subInfo["node"])
else:
self.fail("Dataset A was subscribed to a wrong site %s" % site)
# Now check /BogusPrimary/CRUZET11-v1/RAW
# According to the spec, this is not custodial anywhere
# Non-custodial at T1_UK_RAL and T2_CH_CERN
# Request only at both sites and with high priority
self.assertTrue(self.testDatasetB in subscriptions, "Dataset B was not subscribed")
subInfoB = subscriptions[self.testDatasetB]
self.assertEqual(len(subInfoB), 2, "Dataset B was not subscribed to all sites")
for subInfo in subInfoB:
site = subInfo["node"]
self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription")
if site == "T1_UK_RAL_MSS" or site == "T2_CH_CERN":
self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset B at %s" % subInfo["node"])
self.assertEqual(
subInfo["request_only"], "y", "Wrong requestOnly for dataset B at %s" % subInfo["node"]
)
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset B at %s" % subInfo["node"])
else:
self.fail("Dataset B was subscribed to a wrong site %s" % site)
myThread = threading.currentThread()
result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset_subscription where subscribed = 1")[
0
].fetchall()
self.assertEqual(result[0][0], 5, "Not all datasets were marked as subscribed")
result = myThread.dbi.processData("SELECT site FROM dbsbuffer_dataset_subscription where subscribed = 0")[
0
].fetchall()
self.assertEqual(result[0][0], "T1_IT_CNAF", "A non-valid CMS site was subscribed")
# Reset and run again and make sure that no duplicate subscriptions are created
myThread.dbi.processData("UPDATE dbsbuffer_dataset_subscription SET subscribed = 0")
subscriber.subscribeDatasets()
self.assertEqual(len(subscriptions[self.testDatasetA]), 3)
self.assertEqual(len(subscriptions[self.testDatasetB]), 2)
return
class ProcessPoolTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase = True)
self.testInit.setSchema(customModules = ["WMCore.Agent.Database"],
useDefault = False)
return
def tearDown(self):
"""
_tearDown_
"""
self.testInit.clearDatabase()
return
def testA_ProcessPool(self):
"""
_testProcessPool_
"""
raise nose.SkipTest
config = self.testInit.getConfiguration()
config.Agent.useHeartbeat = False
self.testInit.generateWorkDir(config)
processPool = ProcessPool("ProcessPool_t.ProcessPoolTestWorker",
totalSlaves = 1,
componentDir = config.General.workDir,
config = config,
namespace = "WMCore_t")
processPool.enqueue(["One", "Two", "Three"])
result = processPool.dequeue(3)
self.assertEqual(len(result), 3, "Error: Expected three items back.")
self.assertTrue( "One" in result)
self.assertTrue( "Two" in result)
self.assertTrue( "Three" in result)
return
def testB_ProcessPoolStress(self):
"""
_testProcessPoolStress_
"""
raise nose.SkipTest
config = self.testInit.getConfiguration()
config.Agent.useHeartbeat = False
self.testInit.generateWorkDir(config)
processPool = ProcessPool("ProcessPool_t.ProcessPoolTestWorker",
totalSlaves = 1,
componentDir = config.General.workDir,
namespace = "WMCore_t",
config = config)
result = None
input = None
for i in range(1000):
input = []
while i > 0:
input.append("COMMAND%s" % i)
i -= 1
processPool.enqueue(input)
result = processPool.dequeue(len(input))
self.assertEqual(len(result), len(input),
"Error: Wrong number of results returned.")
for k in result:
self.assertTrue(k in input)
return
def testC_MultiPool(self):
"""
_testMultiPool_
Run a test with multiple workers
"""
raise nose.SkipTest
config = self.testInit.getConfiguration()
config.Agent.useHeartbeat = False
self.testInit.generateWorkDir(config)
processPool = ProcessPool("ProcessPool_t.ProcessPoolTestWorker",
totalSlaves = 3,
componentDir = config.General.workDir,
namespace = "WMCore_t",
config = config)
for i in range(100):
input = []
while i > 0:
input.append("COMMAND%s" % i)
i -= 1
processPool.enqueue(input)
result = processPool.dequeue(len(input))
self.assertEqual(len(result), len(input),
"Error: Wrong number of results returned.")
class HarnessTest(unittest.TestCase):
"""
TestCase for TestComponent module
"""
tempDir = None
def setUp(self):
"""
setup for test.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema()
def tearDown(self):
"""
Delete database
"""
self.testInit.clearDatabase()
def testB(self):
raise nose.SkipTest
config = self.testInit.getConfiguration()
self.tempDir = self.testInit.generateWorkDir(config)
config.component_("TestComponent")
config.TestComponent.logLevel = 'INFO'
config.section_("General")
config.TestComponent.componentDir = os.path.join( \
self.tempDir, "Components/TestComponent1")
config.General.workDir = config.TestComponent.componentDir
os.makedirs(config.TestComponent.componentDir)
# as this is a test we build the string from our global environment
# parameters normally you put this straight into the DefaultConfig.py file:
# testInit.getConfiguration returns from the environment variable by default
testComponent = TestComponent(config)
testComponent.prepareToStart()
testComponent.handleMessage('LogState', '')
testComponent.handleMessage('TestMessage1', 'TestMessag1Payload')
testComponent.handleMessage('TestMessage2', 'TestMessag2Payload')
testComponent.handleMessage('TestMessage3', 'TestMessag3Payload')
testComponent.handleMessage('TestMessage4', 'TestMessag4Payload')
testComponent.handleMessage('Logging.DEBUG', '')
testComponent.handleMessage('Logging.WARNING', '')
testComponent.handleMessage('Logging.CRITICAL', '')
testComponent.handleMessage('Logging.ERROR', '')
testComponent.handleMessage('Logging.INFO', '')
testComponent.handleMessage('Logging.SQLDEBUG', '')
testComponent.handleMessage('TestComponent:Logging.DEBUG', '')
testComponent.handleMessage('TestComponent:Logging.WARNING', '')
testComponent.handleMessage('TestComponent:Logging.CRITICAL', '')
testComponent.handleMessage('TestComponent:Logging.ERROR', '')
testComponent.handleMessage('TestComponent:Logging.INFO', '')
testComponent.handleMessage('TestComponent:Logging.SQLDEBUG', '')
# test a non existing message (to generate an error)
errorMsg = ''
try:
testComponent.handleMessage('NonExistingMessageType', '')
except Exception, ex:
errorMsg = str(ex)
self.assertTrue(
errorMsg.startswith('Message NonExistingMessageType with payload'))
