def testA_basicUploadTest(self):
"""
_basicUploadTest_
Do everything simply once
Create dataset, algo, files, blocks,
upload them,
mark as done, finish them, migrate them
Also check the timeout
"""
myThread = threading.currentThread()
config = self.createConfig()
self.injectWorkflow(MaxWaitTime=3)
config.DBSUpload.pollInterval = 4
name = "ThisIsATest_%s" % (makeUUID())
tier = "RECO"
nFiles = 12
files = self.getFiles(name=name, tier=tier, nFiles=nFiles)
datasetPath = '/%s/%s/%s' % (name, name, tier)
# Load components that are necessary to check status
factory = WMFactory("dbsUpload",
"WMComponent.DBSUpload.Database.Interface")
dbinterface = factory.loadObject("UploadToDBS")
dbsInterface = DBSInterface(config=config)
localAPI = dbsInterface.getAPIRef()
globeAPI = dbsInterface.getAPIRef(globalRef=True)
# In the first round we should create blocks for the first dataset
# The child dataset should not be handled until the parent is uploaded
testDBSUpload = DBSUploadPoller(config=config)
testDBSUpload.algorithm()
# First, see if there are any blocks
# One in DBS, one not in DBS
result = myThread.dbi.processData(
"SELECT status FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(len(result), 2)
self.assertEqual(result, [('InGlobalDBS', ), ('Open', )])
# Check to see if datasets and algos are in local DBS
result = listAlgorithms(apiRef=localAPI, patternExe=name)
self.assertEqual(len(result), 1)
self.assertEqual(result[0]['ExecutableName'], name)
result = listPrimaryDatasets(apiRef=localAPI, match=name)
self.assertEqual(result, [name])
result = listProcessedDatasets(apiRef=localAPI,
primary=name,
dataTier="*")
# Then check and see that the closed block made it into local DBS
affectedBlocks = listBlocks(apiRef=localAPI, datasetPath=datasetPath)
if affectedBlocks[0]['OpenForWriting'] == '0':
self.assertEqual(affectedBlocks[1]['OpenForWriting'], '1')
self.assertEqual(affectedBlocks[0]['NumberOfFiles'], 10)
self.assertEqual(affectedBlocks[1]['NumberOfFiles'], 2)
else:
self.assertEqual(affectedBlocks[0]['OpenForWriting'], '1')
self.assertEqual(affectedBlocks[1]['NumberOfFiles'], 10)
self.assertEqual(affectedBlocks[0]['NumberOfFiles'], 2)
# Check to make sure all the files are in local
result = listDatasetFiles(apiRef=localAPI, datasetPath=datasetPath)
fileLFNs = [x['lfn'] for x in files]
for lfn in fileLFNs:
self.assertTrue(lfn in result)
# Make sure the child files aren't there
flag = False
try:
listDatasetFiles(apiRef=localAPI,
datasetPath='/%s/%s_2/%s' % (name, name, tier))
except Exception as ex:
flag = True
self.assertTrue(flag)
# There should be one blocks in global
# It should have ten files and be closed
result = listBlocks(apiRef=globeAPI, datasetPath=datasetPath)
self.assertEqual(len(result), 1)
for block in result:
self.assertEqual(block['OpenForWriting'], '0')
self.assertTrue(block['NumberOfFiles'] in [2, 10])
# Okay, deep breath. First round done
# In the second round, the second block of the parent fileset should transfer
# Make sure that the timeout functions work
time.sleep(10)
testDBSUpload.algorithm()
result = myThread.dbi.processData(
"SELECT status FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(len(result), 2)
self.assertEqual(result, [('InGlobalDBS', ), ('InGlobalDBS', )])
# Check to make sure all the files are in global
result = listDatasetFiles(apiRef=globeAPI, datasetPath=datasetPath)
for lfn in fileLFNs:
self.assertTrue(lfn in result)
# Make sure the child files aren't there
flag = False
try:
listDatasetFiles(apiRef=localAPI,
datasetPath='/%s/%s_2/%s' % (name, name, tier))
except Exception as ex:
flag = True
self.assertTrue(flag)
# Third round
# Both of the parent blocks should have transferred
# So the child block should now transfer
testDBSUpload.algorithm()
result = myThread.dbi.processData(
"SELECT status FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(result, [('InGlobalDBS', ), ('InGlobalDBS', ),
('Open', )])
flag = False
try:
result = listDatasetFiles(apiRef=localAPI,
datasetPath='/%s/%s_2/%s' %
(name, name, tier))
except Exception as ex:
flag = True
self.assertFalse(flag)
self.assertEqual(len(result), 1)
return
def testC_FailTest(self):
"""
_FailTest_
THIS TEST IS DANGEROUS!
Figure out what happens when we trigger rollbacks
"""
myThread = threading.currentThread()
config = self.createConfig()
config.DBSUpload.abortStepTwo = True
originalOut = sys.stdout
originalErr = sys.stderr
dbsInterface = DBSInterface(config=config)
localAPI = dbsInterface.getAPIRef()
globeAPI = dbsInterface.getAPIRef(globalRef=True)
name = "ThisIsATest_%s" % (makeUUID())
tier = "RECO"
nFiles = 12
files = self.getFiles(name=name, tier=tier, nFiles=nFiles)
datasetPath = '/%s/%s/%s' % (name, name, tier)
testDBSUpload = DBSUploadPoller(config=config)
try:
testDBSUpload.algorithm()
except Exception as ex:
pass
# Aborting in step two should result in no results
result = myThread.dbi.processData(
"SELECT status FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(len(result), 0)
config.DBSUpload.abortStepTwo = False
config.DBSUpload.abortStepThree = True
testDBSUpload = DBSUploadPoller(config=config)
try:
testDBSUpload.algorithm()
except Exception as ex:
pass
result = myThread.dbi.processData(
"SELECT status FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(result, [('Pending', ), ('Open', )])
result = myThread.dbi.processData(
"SELECT status FROM dbsbuffer_file WHERE dataset_algo = 1"
)[0].fetchall()
for res in result:
self.assertEqual(res[0], 'READY')
config.DBSUpload.abortStepThree = False
self.injectWorkflow(MaxWaitTime=300)
testDBSUpload = DBSUploadPoller(config=config)
testDBSUpload.algorithm()
# After this, one block should have been uploaded, one should still be open
# This is the result of the pending block updating, and the open block staying open
result = myThread.dbi.processData(
"SELECT status, id FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(result, [('InGlobalDBS', 3), ('Open', 4)])
# Check that one block got there
result = listBlocks(apiRef=globeAPI, datasetPath=datasetPath)
self.assertEqual(len(result), 1)
self.assertEqual(result[0]['NumberOfFiles'], 10)
self.assertEqual(result[0]['NumberOfEvents'], 200)
self.assertEqual(result[0]['BlockSize'], 10240)
# Check that ten files got there
result = listDatasetFiles(apiRef=globeAPI, datasetPath=datasetPath)
self.assertEqual(len(result), 10)
myThread.dbi.processData(
"UPDATE dbsbuffer_workflow SET block_close_max_wait_time = 1")
testDBSUpload = DBSUploadPoller(config=config)
time.sleep(3)
testDBSUpload.algorithm()
result = myThread.dbi.processData(
"SELECT status FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(result, [('InGlobalDBS', ), ('InGlobalDBS', )])
result = listDatasetFiles(apiRef=globeAPI, datasetPath=datasetPath)
self.assertEqual(len(result), 12)
fileLFNs = [x['lfn'] for x in files]
for lfn in fileLFNs:
self.assertTrue(lfn in result)
testDBSUpload.algorithm()
result = myThread.dbi.processData(
"SELECT status FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(result, [('InGlobalDBS', ), ('InGlobalDBS', ),
('Open', )])
time.sleep(5)
testDBSUpload.algorithm()
time.sleep(2)
result = myThread.dbi.processData(
"SELECT status FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(result, [('InGlobalDBS', ), ('InGlobalDBS', ),
('InGlobalDBS', )])
result = listDatasetFiles(apiRef=globeAPI,
datasetPath='/%s/%s_2/%s' %
(name, name, tier))
self.assertEqual(len(result), 1)
sys.stdout = originalOut
sys.stderr = originalErr
return
def testA_basicUploadTest(self):
"""
_basicUploadTest_
Do everything simply once
Create dataset, algo, files, blocks,
upload them,
mark as done, finish them, migrate them
Also check the timeout
"""
myThread = threading.currentThread()
config = self.createConfig()
self.injectWorkflow(MaxWaitTime = 3)
config.DBSUpload.pollInterval = 4
name = "ThisIsATest_%s" % (makeUUID())
tier = "RECO"
nFiles = 12
files = self.getFiles(name = name, tier = tier, nFiles = nFiles)
datasetPath = '/%s/%s/%s' % (name, name, tier)
# Load components that are necessary to check status
factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database.Interface")
dbinterface = factory.loadObject("UploadToDBS")
dbsInterface = DBSInterface(config = config)
localAPI = dbsInterface.getAPIRef()
globeAPI = dbsInterface.getAPIRef(globalRef = True)
# In the first round we should create blocks for the first dataset
# The child dataset should not be handled until the parent is uploaded
testDBSUpload = DBSUploadPoller(config = config)
testDBSUpload.algorithm()
# First, see if there are any blocks
# One in DBS, one not in DBS
result = myThread.dbi.processData("SELECT status FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(len(result), 2)
self.assertEqual(result, [('InGlobalDBS',), ('Open',)])
# Check to see if datasets and algos are in local DBS
result = listAlgorithms(apiRef = localAPI, patternExe = name)
self.assertEqual(len(result), 1)
self.assertEqual(result[0]['ExecutableName'], name)
result = listPrimaryDatasets(apiRef = localAPI, match = name)
self.assertEqual(result, [name])
result = listProcessedDatasets(apiRef = localAPI, primary = name, dataTier = "*")
# Then check and see that the closed block made it into local DBS
affectedBlocks = listBlocks(apiRef = localAPI, datasetPath = datasetPath)
if affectedBlocks[0]['OpenForWriting'] == '0':
self.assertEqual(affectedBlocks[1]['OpenForWriting'], '1')
self.assertEqual(affectedBlocks[0]['NumberOfFiles'], 10)
self.assertEqual(affectedBlocks[1]['NumberOfFiles'], 2)
else:
self.assertEqual(affectedBlocks[0]['OpenForWriting'], '1')
self.assertEqual(affectedBlocks[1]['NumberOfFiles'], 10)
self.assertEqual(affectedBlocks[0]['NumberOfFiles'], 2)
# Check to make sure all the files are in local
result = listDatasetFiles(apiRef = localAPI, datasetPath = datasetPath)
fileLFNs = [x['lfn'] for x in files]
for lfn in fileLFNs:
self.assertTrue(lfn in result)
# Make sure the child files aren't there
flag = False
try:
listDatasetFiles(apiRef = localAPI,
datasetPath = '/%s/%s_2/%s' % (name, name, tier))
except Exception, ex:
flag = True
except Exception, ex:
flag = True
self.assertTrue(flag)
# Third round
# Both of the parent blocks should have transferred
# So the child block should now transfer
testDBSUpload.algorithm()
result = myThread.dbi.processData("SELECT status FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(result, [('InGlobalDBS',), ('InGlobalDBS',), ('Open',)])
flag = False
try:
result = listDatasetFiles(apiRef = localAPI,
datasetPath = '/%s/%s_2/%s' % (name, name, tier))
except Exception, ex:
flag = True
self.assertFalse(flag)
self.assertEqual(len(result), 1)
return
@attr('integration')
def testB_AlgoMigration(self):
"""
_AlgoMigration_
Test our ability to migrate multiple algos to global
class DBSUploadTest(unittest.TestCase):
"""
_DBSUploadTest_
TestCase for DBSUpload module
"""
_maxMessage = 10
def setUp(self):
"""
_setUp_
setUp function for unittest
"""
# Set constants
self.couchDB = "config_test"
self.configURL = "RANDOM;;URL;;NAME"
self.configString = "This is a random string"
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(
customModules=["WMComponent.DBS3Buffer", 'WMCore.Agent.Database'],
useDefault=False)
self.testInit.setupCouch(self.couchDB, "GroupUser", "ConfigCache")
myThread = threading.currentThread()
self.bufferFactory = DAOFactory(
package="WMComponent.DBSBuffer.Database",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.buffer3Factory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
locationAction = self.bufferFactory(
classname="DBSBufferFiles.AddLocation")
locationAction.execute(siteName="se1.cern.ch")
locationAction.execute(siteName="se1.fnal.gov")
locationAction.execute(siteName="malpaquet")
# Set heartbeat
self.componentName = 'JobSubmitter'
self.heartbeatAPI = HeartbeatAPI(self.componentName)
self.heartbeatAPI.registerComponent()
# Set up a config cache
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName=self.couchDB)
configCache.createUserGroup(groupname="testGroup", username='******')
self.testDir = self.testInit.generateWorkDir()
psetPath = os.path.join(self.testDir, "PSet.txt")
f = open(psetPath, 'w')
f.write(self.configString)
f.close()
configCache.addConfig(newConfig=psetPath, psetHash=None)
configCache.save()
self.configURL = "%s;;%s;;%s" % (os.environ["COUCHURL"], self.couchDB,
configCache.getCouchID())
return
def tearDown(self):
"""
_tearDown_
tearDown function for unittest
"""
self.testInit.clearDatabase()
self.testInit.tearDownCouch()
self.testInit.delWorkDir()
return
def createConfig(self):
"""
_createConfig_
This creates the actual config file used by the component
"""
config = Configuration()
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = 'DBSUpload'
config.Agent.useHeartbeat = False
#Now the CoreDatabase information
#This should be the dialect, dburl, etc
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.component_("DBSUpload")
config.DBSUpload.pollInterval = 10
config.DBSUpload.logLevel = 'ERROR'
config.DBSUpload.maxThreads = 1
config.DBSUpload.namespace = 'WMComponent.DBSUpload.DBSUpload'
config.DBSUpload.componentDir = os.path.join(os.getcwd(), 'Components')
config.DBSUpload.workerThreads = 4
config.section_("DBSInterface")
config.DBSInterface.globalDBSUrl = 'http://vocms09.cern.ch:8880/cms_dbs_int_local_xx_writer/servlet/DBSServlet'
config.DBSInterface.globalDBSVersion = 'DBS_2_0_9'
config.DBSInterface.DBSUrl = 'http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet'
config.DBSInterface.DBSVersion = 'DBS_2_0_9'
config.DBSInterface.MaxFilesToCommit = 10
# addition for Alerts messaging framework, work (alerts) and control
# channel addresses to which the component will be sending alerts
# these are destination addresses where AlertProcessor:Receiver listens
config.section_("Alert")
config.Alert.address = "tcp://127.0.0.1:5557"
config.Alert.controlAddr = "tcp://127.0.0.1:5559"
# configure threshold of DBS upload queue size alert threshold
# reference: trac ticket #1628
config.DBSUpload.alertUploadQueueSize = 2000
return config
def injectWorkflow(self,
workflowName='TestWorkflow',
taskPath='/TestWorkflow/ReadingEvents',
MaxWaitTime=10000,
MaxFiles=10,
MaxEvents=250000000,
MaxSize=9999999999):
"""
_injectWorklow_
Inject a dummy worklow in DBSBuffer for testing,
returns the workflow ID
"""
injectWorkflowDAO = self.buffer3Factory("InsertWorkflow")
workflowID = injectWorkflowDAO.execute(workflowName, taskPath,
MaxWaitTime, MaxFiles,
MaxEvents, MaxSize)
return workflowID
def getFiles(self,
name,
tier,
nFiles=12,
site="malpaquet",
workflowName=None,
taskPath=None,
noChild=False):
"""
Create some quick dummy test files
"""
if workflowName is not None and taskPath is not None:
workflowId = self.injectWorkflow(workflowName=workflowName,
taskPath=taskPath)
else:
workflowId = self.injectWorkflow()
files = []
for f in range(0, nFiles):
testFile = DBSBufferFile(lfn='%s-%s-%i' % (name, site, f),
size=1024,
events=20,
checksums={'cksum': 1},
workflowId=workflowId)
testFile.setAlgorithm(appName=name,
appVer="CMSSW_3_1_1",
appFam="RECO",
psetHash="GIBBERISH",
configContent=self.configURL)
testFile.setDatasetPath("/%s/%s/%s" % (name, name, tier))
testFile.addRun(Run(1, *[f]))
testFile.setGlobalTag("aGlobalTag")
testFile.create()
testFile.setLocation(site)
files.append(testFile)
if not noChild:
testFileChild = DBSBufferFile(lfn='%s-%s-child' % (name, site),
size=1024,
events=10,
checksums={'cksum': 1},
workflowId=workflowId)
testFileChild.setAlgorithm(appName=name,
appVer="CMSSW_3_1_1",
appFam="RECO",
psetHash="GIBBERISH",
configContent=self.configURL)
testFileChild.setDatasetPath("/%s/%s_2/RECO" % (name, name))
testFileChild.addRun(Run(1, *[45]))
testFileChild.setGlobalTag("aGlobalTag")
testFileChild.create()
testFileChild.setLocation(site)
testFileChild.addParents([x['lfn'] for x in files])
return files
@attr('integration')
def testA_basicUploadTest(self):
"""
_basicUploadTest_
Do everything simply once
Create dataset, algo, files, blocks,
upload them,
mark as done, finish them, migrate them
Also check the timeout
"""
myThread = threading.currentThread()
config = self.createConfig()
self.injectWorkflow(MaxWaitTime=3)
config.DBSUpload.pollInterval = 4
name = "ThisIsATest_%s" % (makeUUID())
tier = "RECO"
nFiles = 12
files = self.getFiles(name=name, tier=tier, nFiles=nFiles)
datasetPath = '/%s/%s/%s' % (name, name, tier)
# Load components that are necessary to check status
factory = WMFactory("dbsUpload",
"WMComponent.DBSUpload.Database.Interface")
dbinterface = factory.loadObject("UploadToDBS")
dbsInterface = DBSInterface(config=config)
localAPI = dbsInterface.getAPIRef()
globeAPI = dbsInterface.getAPIRef(globalRef=True)
# In the first round we should create blocks for the first dataset
# The child dataset should not be handled until the parent is uploaded
testDBSUpload = DBSUploadPoller(config=config)
testDBSUpload.algorithm()
# First, see if there are any blocks
# One in DBS, one not in DBS
result = myThread.dbi.processData(
"SELECT status FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(len(result), 2)
self.assertEqual(result, [('InGlobalDBS', ), ('Open', )])
# Check to see if datasets and algos are in local DBS
result = listAlgorithms(apiRef=localAPI, patternExe=name)
self.assertEqual(len(result), 1)
self.assertEqual(result[0]['ExecutableName'], name)
result = listPrimaryDatasets(apiRef=localAPI, match=name)
self.assertEqual(result, [name])
result = listProcessedDatasets(apiRef=localAPI,
primary=name,
dataTier="*")
# Then check and see that the closed block made it into local DBS
affectedBlocks = listBlocks(apiRef=localAPI, datasetPath=datasetPath)
if affectedBlocks[0]['OpenForWriting'] == '0':
self.assertEqual(affectedBlocks[1]['OpenForWriting'], '1')
self.assertEqual(affectedBlocks[0]['NumberOfFiles'], 10)
self.assertEqual(affectedBlocks[1]['NumberOfFiles'], 2)
else:
self.assertEqual(affectedBlocks[0]['OpenForWriting'], '1')
self.assertEqual(affectedBlocks[1]['NumberOfFiles'], 10)
self.assertEqual(affectedBlocks[0]['NumberOfFiles'], 2)
# Check to make sure all the files are in local
result = listDatasetFiles(apiRef=localAPI, datasetPath=datasetPath)
fileLFNs = [x['lfn'] for x in files]
for lfn in fileLFNs:
self.assertTrue(lfn in result)
# Make sure the child files aren't there
flag = False
try:
listDatasetFiles(apiRef=localAPI,
datasetPath='/%s/%s_2/%s' % (name, name, tier))
except Exception, ex:
flag = True
self.assertTrue(flag)
# There should be one blocks in global
# It should have ten files and be closed
result = listBlocks(apiRef=globeAPI, datasetPath=datasetPath)
self.assertEqual(len(result), 1)
for block in result:
self.assertEqual(block['OpenForWriting'], '0')
self.assertTrue(block['NumberOfFiles'] in [2, 10])
# Okay, deep breath. First round done
# In the second round, the second block of the parent fileset should transfer
# Make sure that the timeout functions work
time.sleep(10)
testDBSUpload.algorithm()
result = myThread.dbi.processData(
"SELECT status FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(len(result), 2)
self.assertEqual(result, [('InGlobalDBS', ), ('InGlobalDBS', )])
# Check to make sure all the files are in global
result = listDatasetFiles(apiRef=globeAPI, datasetPath=datasetPath)
for lfn in fileLFNs:
self.assertTrue(lfn in result)
# Make sure the child files aren't there
flag = False
try:
listDatasetFiles(apiRef=localAPI,
datasetPath='/%s/%s_2/%s' % (name, name, tier))
except Exception, ex:
flag = True
self.assertTrue(flag)
# Third round
# Both of the parent blocks should have transferred
# So the child block should now transfer
testDBSUpload.algorithm()
result = myThread.dbi.processData(
"SELECT status FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(result, [('InGlobalDBS', ), ('InGlobalDBS', ),
('Open', )])
flag = False
try:
result = listDatasetFiles(apiRef=localAPI,
datasetPath='/%s/%s_2/%s' %
(name, name, tier))
except Exception, ex:
flag = True
self.assertFalse(flag)
self.assertEqual(len(result), 1)
return
@attr('integration')
def testB_AlgoMigration(self):
"""
_AlgoMigration_
Test our ability to migrate multiple algos to global
def testC_FailTest(self):
"""
_FailTest_
THIS TEST IS DANGEROUS!
Figure out what happens when we trigger rollbacks
"""
myThread = threading.currentThread()
config = self.createConfig()
config.DBSUpload.abortStepTwo = True
originalOut = sys.stdout
originalErr = sys.stderr
dbsInterface = DBSInterface(config = config)
localAPI = dbsInterface.getAPIRef()
globeAPI = dbsInterface.getAPIRef(globalRef = True)
name = "ThisIsATest_%s" % (makeUUID())
tier = "RECO"
nFiles = 12
files = self.getFiles(name = name, tier = tier, nFiles = nFiles)
datasetPath = '/%s/%s/%s' % (name, name, tier)
testDBSUpload = DBSUploadPoller(config = config)
try:
testDBSUpload.algorithm()
except Exception as ex:
pass
# Aborting in step two should result in no results
result = myThread.dbi.processData("SELECT status FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(len(result), 0)
config.DBSUpload.abortStepTwo = False
config.DBSUpload.abortStepThree = True
testDBSUpload = DBSUploadPoller(config = config)
try:
testDBSUpload.algorithm()
except Exception as ex:
pass
result = myThread.dbi.processData("SELECT status FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(result, [('Pending',), ('Open',)])
result = myThread.dbi.processData("SELECT status FROM dbsbuffer_file WHERE dataset_algo = 1")[0].fetchall()
for res in result:
self.assertEqual(res[0], 'READY')
config.DBSUpload.abortStepThree = False
self.injectWorkflow(MaxWaitTime = 300)
testDBSUpload = DBSUploadPoller(config = config)
testDBSUpload.algorithm()
# After this, one block should have been uploaded, one should still be open
# This is the result of the pending block updating, and the open block staying open
result = myThread.dbi.processData("SELECT status, id FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(result, [('InGlobalDBS', 3L), ('Open', 4L)])
# Check that one block got there
result = listBlocks(apiRef = globeAPI, datasetPath = datasetPath)
self.assertEqual(len(result), 1)
self.assertEqual(result[0]['NumberOfFiles'], 10)
self.assertEqual(result[0]['NumberOfEvents'], 200)
self.assertEqual(result[0]['BlockSize'], 10240)
# Check that ten files got there
result = listDatasetFiles(apiRef = globeAPI, datasetPath = datasetPath)
self.assertEqual(len(result), 10)
myThread.dbi.processData("UPDATE dbsbuffer_workflow SET block_close_max_wait_time = 1")
testDBSUpload = DBSUploadPoller(config = config)
time.sleep(3)
testDBSUpload.algorithm()
result = myThread.dbi.processData("SELECT status FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(result, [('InGlobalDBS',), ('InGlobalDBS',)])
result = listDatasetFiles(apiRef = globeAPI, datasetPath = datasetPath)
self.assertEqual(len(result), 12)
fileLFNs = [x['lfn'] for x in files]
for lfn in fileLFNs:
self.assertTrue(lfn in result)
testDBSUpload.algorithm()
result = myThread.dbi.processData("SELECT status FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(result, [('InGlobalDBS',), ('InGlobalDBS',), ('Open',)])
time.sleep(5)
testDBSUpload.algorithm()
time.sleep(2)
result = myThread.dbi.processData("SELECT status FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(result, [('InGlobalDBS',), ('InGlobalDBS',), ('InGlobalDBS',)])
result = listDatasetFiles(apiRef = globeAPI,
datasetPath = '/%s/%s_2/%s' % (name, name, tier))
self.assertEqual(len(result), 1)
sys.stdout = originalOut
sys.stderr = originalErr
return
