def __init__(self):
"""
__init__
Init the DB
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase=True)
self.testInit.setSchema(customModules=["WMComponent.DBS3Buffer"],
useDefault=False)
self.configFile = EmulatorSetup.setupWMAgentConfig()
myThread = threading.currentThread()
self.bufferFactory = DAOFactory(
package="WMComponent.DBSBuffer.Database",
logger=myThread.logger,
dbinterface=myThread.dbi)
locationAction = self.bufferFactory(
classname="DBSBufferFiles.AddLocation")
locationAction.execute(siteName="se1.cern.ch")
locationAction.execute(siteName="se1.fnal.gov")
locationAction.execute(siteName="malpaquet")
config = self.getConfig()
self.dbsUploader = DBSUploadPoller(config=config)
return
def preInitialization(self):
print("DBS3Upload.preInitialization")
# Add event loop to worker manager
myThread = threading.currentThread()
pollInterval = self.config.DBS3Upload.pollInterval
logging.info("Setting poll interval to %s seconds" % pollInterval)
myThread.workerThreadManager.addWorker(DBSUploadPoller(self.config), pollInterval)
return
def testBasicUpload(self):
"""
_testBasicUpload_
Verify that we can successfully upload to DBS3. Also verify that the
uploader correctly handles files parentage when uploading.
"""
self.dbsApi = DbsApi(url = self.dbsUrl)
config = self.getConfig()
dbsUploader = DBSUploadPoller(config = config)
# First test verifies that uploader will poll and then not do anything
# as the database is empty.
dbsUploader.algorithm()
acqEra = "Summer%s" % (int(time.time()))
parentFiles = self.createParentFiles(acqEra)
# The algorithm needs to be run twice. On the first iteration it will
# create all the blocks and upload one. On the second iteration it will
# timeout and upload the second block.
dbsUploader.algorithm()
time.sleep(5)
dbsUploader.algorithm()
time.sleep(5)
# Verify the files made it into DBS3.
self.verifyData(parentFiles[0]["datasetPath"], parentFiles)
# Inject some more parent files and some child files into DBSBuffer.
# Run the uploader twice, only the parent files should be added to DBS3.
(moreParentFiles, childFiles) = \
self.createFilesWithChildren(parentFiles, acqEra)
dbsUploader.algorithm()
time.sleep(5)
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(parentFiles[0]["datasetPath"],
parentFiles + moreParentFiles)
# Run the uploader another two times to upload the child files. Verify
# that the child files were uploaded.
dbsUploader.algorithm()
time.sleep(5)
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(childFiles[0]["datasetPath"], childFiles)
return
def testDualUpload(self):
"""
_testDualUpload_
Verify that the dual upload mode works correctly.
"""
self.dbsApi = DbsApi(url = self.dbsUrl)
config = self.getConfig()
dbsUploader = DBSUploadPoller(config = config)
dbsUtil = DBSBufferUtil()
# First test verifies that uploader will poll and then not do anything
# as the database is empty.
dbsUploader.algorithm()
acqEra = "Summer%s" % (int(time.time()))
parentFiles = self.createParentFiles(acqEra)
(moreParentFiles, childFiles) = \
self.createFilesWithChildren(parentFiles, acqEra)
allFiles = parentFiles + moreParentFiles
allBlocks = []
for i in range(4):
DBSBufferDataset(parentFiles[0]["datasetPath"]).create()
blockName = parentFiles[0]["datasetPath"] + "#" + makeUUID()
dbsBlock = DBSBufferBlock(blockName,
location = "malpaquet",
datasetpath = None)
dbsBlock.status = "Open"
dbsBlock.setDataset(parentFiles[0]["datasetPath"], 'data', 'VALID')
dbsUtil.createBlocks([dbsBlock])
for file in allFiles[i * 5 : (i * 5) + 5]:
dbsBlock.addFile(file, 'data', 'VALID')
dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]})
if i < 2:
dbsBlock.status = "InDBS"
dbsUtil.updateBlocks([dbsBlock])
dbsUtil.updateFileStatus([dbsBlock], "InDBS")
allBlocks.append(dbsBlock)
DBSBufferDataset(childFiles[0]["datasetPath"]).create()
blockName = childFiles[0]["datasetPath"] + "#" + makeUUID()
dbsBlock = DBSBufferBlock(blockName,
location = "malpaquet",
datasetpath = None)
dbsBlock.status = "InDBS"
dbsBlock.setDataset(childFiles[0]["datasetPath"], 'data', 'VALID')
dbsUtil.createBlocks([dbsBlock])
for file in childFiles:
dbsBlock.addFile(file, 'data', 'VALID')
dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]})
dbsUtil.updateFileStatus([dbsBlock], "InDBS")
dbsUploader.algorithm()
time.sleep(5)
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(parentFiles[0]["datasetPath"], parentFiles)
# Change the status of the rest of the parent blocks so we can upload
# them and the children.
for dbsBlock in allBlocks:
dbsBlock.status = "InDBS"
dbsUtil.updateBlocks([dbsBlock])
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(parentFiles[0]["datasetPath"], parentFiles + moreParentFiles)
# Run the uploader one more time to upload the children.
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(childFiles[0]["datasetPath"], childFiles)
return
def testA_basicFunction(self):
"""
_basicFunction_
See if I can make the damn thing work.
"""
myThread = threading.currentThread()
config = self.getConfig()
from WMComponent.DBS3Buffer.DBSUploadPoller import DBSUploadPoller
dbsUploader = DBSUploadPoller(config=config)
dbsUtil = DBSBufferUtil()
from dbs.apis.dbsClient import DbsApi
dbsApi = DbsApi(url=config.DBSUpload.dbsUrl)
# This should do nothing
# Just making sure we don't crash
try:
dbsUploader.algorithm()
except:
dbsUploader.close()
raise
name = "ThisIsATest%s" % (int(time.time()))
tier = "RECO"
nFiles = 12
name = name.replace('-', '_')
name = '%s-v0' % name
files = self.getFiles(name=name, tier=tier, nFiles=nFiles)
datasetPath = "/Cosmics/%s/%s" % (name, tier)
try:
dbsUploader.algorithm()
except:
dbsUploader.close()
raise
time.sleep(5)
# Now look in DBS
try:
result = dbsApi.listDatasets(dataset=datasetPath,
detail=True,
dataset_access_type='PRODUCTION')
self.assertEqual(len(result), 1)
self.assertEqual(result[0]['data_tier_name'], 'RECO')
self.assertEqual(result[0]['processing_version'], 0)
self.assertEqual(result[0]['acquisition_era_name'],
name.split('-')[0])
result = dbsApi.listFiles(dataset=datasetPath)
self.assertEqual(len(result), 11)
except:
dbsUploader.close()
raise
# All the blocks except for the last one should
# now be there
result = myThread.dbi.processData(
"SELECT id FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(len(result), 12)
# The last block should still be open
self.assertEqual(len(dbsUtil.findOpenBlocks()), 1)
try:
dbsUploader.algorithm()
except:
raise
finally:
dbsUploader.close()
# All files should now be available
result = dbsApi.listFiles(dataset=datasetPath)
self.assertEqual(len(result), 12)
# The last block should now be closed
self.assertEqual(len(dbsUtil.findOpenBlocks()), 0)
result = myThread.dbi.processData(
"SELECT status FROM dbsbuffer_block")[0].fetchall()
for res in result:
self.assertEqual(res.values()[0], 'InDBS')
return
