def testBasic(self):
"""
_testBasic_
Test the basic functions of the DBSBufferDataset,
create, load, exists and also the ability
to add subscriptions.
"""
originalDataset = DBSBufferDataset(path = '/bogus/bogus/go')
originalDataset.create()
myThread = threading.currentThread()
result = myThread.dbi.processData("SELECT id FROM dbsbuffer_dataset")[0].fetchall()
self.assertEqual(originalDataset.exists(), result[0][0])
duplicateDataset = DBSBufferDataset(path = '/bogus/bogus/go')
duplicateDataset.create()
self.assertEqual(originalDataset.exists(), duplicateDataset.exists())
result = myThread.dbi.processData("SELECT COUNT(id) FROM dbsbuffer_dataset")[0].fetchall()
self.assertEqual(result[0][0], 1)
loadedDataset = DBSBufferDataset(path = '/bogus/bogus/go')
loadedDataset.load()
self.assertEqual(loadedDataset.exists(), originalDataset.exists())
secondDataset = DBSBufferDataset(path = '/BogusPrimary/Run2012Z-PromptReco-v1/RECO')
secondDataset.create()
workload = WMWorkloadHelper()
workload.load(os.path.join(getTestBase(), 'WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl'))
secondDataset.addSubscription(workload.getSubscriptionInformation()['/BogusPrimary/Run2012Z-PromptReco-v1/RECO'])
secondDataset.addSubscription(workload.getSubscriptionInformation()['/BogusPrimary/Run2012Z-PromptReco-v1/RECO'])
self.assertEqual(len(secondDataset['subscriptions']), 3)
result = myThread.dbi.processData("SELECT COUNT(id) FROM dbsbuffer_dataset_subscription")[0].fetchall()
self.assertEqual(result[0][0], 3)
return
def testBasic(self):
"""
_testBasic_
Test the basic functions of the DBSBufferDataset,
create, load, exists and also the ability
to add subscriptions.
"""
originalDataset = DBSBufferDataset(path='/bogus/bogus/go')
originalDataset.create()
myThread = threading.currentThread()
result = myThread.dbi.processData(
"SELECT id FROM dbsbuffer_dataset")[0].fetchall()
self.assertEqual(originalDataset.exists(), result[0][0])
duplicateDataset = DBSBufferDataset(path='/bogus/bogus/go')
duplicateDataset.create()
self.assertEqual(originalDataset.exists(), duplicateDataset.exists())
result = myThread.dbi.processData(
"SELECT COUNT(id) FROM dbsbuffer_dataset")[0].fetchall()
self.assertEqual(result[0][0], 1)
loadedDataset = DBSBufferDataset(path='/bogus/bogus/go')
loadedDataset.load()
self.assertEqual(loadedDataset.exists(), originalDataset.exists())
secondDataset = DBSBufferDataset(
path='/BogusPrimary/Run2012Z-PromptReco-v1/RECO')
secondDataset.create()
workload = WMWorkloadHelper()
workload.load(
os.path.join(
getTestBase(),
'WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl'))
secondDataset.addSubscription(
workload.getSubscriptionInformation()
['/BogusPrimary/Run2012Z-PromptReco-v1/RECO'])
secondDataset.addSubscription(
workload.getSubscriptionInformation()
['/BogusPrimary/Run2012Z-PromptReco-v1/RECO'])
self.assertEqual(len(secondDataset['subscriptions']), 3)
result = myThread.dbi.processData(
"SELECT COUNT(id) FROM dbsbuffer_dataset_subscription"
)[0].fetchall()
self.assertEqual(result[0][0], 3)
return
def stuffDatabase(self, tier0Mode = False):
"""
_stuffDatabase_
Fill the dbsbuffer with some files and blocks. We'll insert a total
of 5 files spanning two blocks. There will be a total of two datasets
inserted into the database.
All files will be already in GLOBAL and in_phedex
"""
myThread = threading.currentThread()
buffer3Factory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
insertWorkflow = buffer3Factory(classname = "InsertWorkflow")
insertWorkflow.execute("BogusRequestA", "BogusTask",
0, 0, 0, 0)
insertWorkflow.execute("BogusRequestB", "BogusTask",
0, 0, 0, 0)
checksums = {"adler32": "1234", "cksum": "5678"}
testFileA = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileA.setDatasetPath(self.testDatasetA)
testFileA.addRun(Run(2, *[45]))
testFileA.create()
testFileB = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileB.setDatasetPath(self.testDatasetA)
testFileB.addRun(Run(2, *[45]))
testFileB.create()
testFileC = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileC.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileC.setDatasetPath(self.testDatasetA)
testFileC.addRun(Run(2, *[45]))
testFileC.create()
self.testFilesA.append(testFileA)
self.testFilesA.append(testFileB)
self.testFilesA.append(testFileC)
testFileD = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileD.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileD.setDatasetPath(self.testDatasetB)
testFileD.addRun(Run(2, *[45]))
testFileD.create()
testFileE = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileE.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileE.setDatasetPath(self.testDatasetB)
testFileE.addRun(Run(2, *[45]))
testFileE.create()
self.testFilesB.append(testFileD)
self.testFilesB.append(testFileE)
uploadFactory = DAOFactory(package = "WMComponent.DBSUpload.Database",
logger = myThread.logger,
dbinterface = myThread.dbi)
createBlock = uploadFactory(classname = "SetBlockStatus")
self.blockAName = self.testDatasetA + "#" + makeUUID()
self.blockBName = self.testDatasetB + "#" + makeUUID()
createBlock.execute(block = self.blockAName, locations = ["srm-cms.cern.ch"], open_status = 'Closed')
createBlock.execute(block = self.blockBName, locations = ["srm-cms.cern.ch"], open_status = 'Closed')
bufferFactory = DAOFactory(package = "WMComponent.DBSBuffer.Database",
logger = myThread.logger,
dbinterface = myThread.dbi)
setBlock = bufferFactory(classname = "DBSBufferFiles.SetBlock")
setBlock.execute(testFileA["lfn"], self.blockAName)
setBlock.execute(testFileB["lfn"], self.blockAName)
setBlock.execute(testFileC["lfn"], self.blockAName)
setBlock.execute(testFileD["lfn"], self.blockBName)
setBlock.execute(testFileE["lfn"], self.blockBName)
fileStatus = bufferFactory(classname = "DBSBufferFiles.SetStatus")
fileStatus.execute(testFileA["lfn"], "GLOBAL")
fileStatus.execute(testFileB["lfn"], "GLOBAL")
fileStatus.execute(testFileC["lfn"], "GLOBAL")
fileStatus.execute(testFileD["lfn"], "GLOBAL")
fileStatus.execute(testFileE["lfn"], "GLOBAL")
phedexStatus = bufferFactory(classname = "DBSBufferFiles.SetPhEDExStatus")
phedexStatus.execute(testFileA["lfn"], 1)
phedexStatus.execute(testFileB["lfn"], 1)
phedexStatus.execute(testFileC["lfn"], 1)
phedexStatus.execute(testFileD["lfn"], 1)
phedexStatus.execute(testFileE["lfn"], 1)
associateWorkflow = buffer3Factory(classname = "DBSBufferFiles.AssociateWorkflowToFile")
associateWorkflow.execute(testFileA["lfn"], "BogusRequestA", "BogusTask")
associateWorkflow.execute(testFileB["lfn"], "BogusRequestA", "BogusTask")
associateWorkflow.execute(testFileC["lfn"], "BogusRequestA", "BogusTask")
associateWorkflow.execute(testFileD["lfn"], "BogusRequestB", "BogusTask")
associateWorkflow.execute(testFileE["lfn"], "BogusRequestB", "BogusTask")
# Make the desired subscriptions
insertSubAction = buffer3Factory(classname = "NewSubscription")
datasetA = DBSBufferDataset(path = self.testDatasetA)
datasetB = DBSBufferDataset(path = self.testDatasetB)
workload = WMWorkloadHelper()
workload.load(os.path.join(getTestBase(), 'WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl'))
if tier0Mode:
# Override the settings
workload.setSubscriptionInformation(custodialSites = ["T0_CH_CERN", "T1_US_FNAL"],
nonCustodialSites = ["T3_CO_Uniandes"],
priority = "Normal", custodialSubType = "Replica",
autoApproveSites = ["T0_CH_CERN"],
dataTier = "RECO")
workload.setSubscriptionInformation(custodialSites = ["T0_CH_CERN", "T1_UK_RAL"],
nonCustodialSites = [],
autoApproveSites = [],
priority = "High", custodialSubType = "Replica",
dataTier = "RAW")
insertSubAction.execute(datasetA.exists(), workload.getSubscriptionInformation()[self.testDatasetA])
insertSubAction.execute(datasetB.exists(), workload.getSubscriptionInformation()[self.testDatasetB])
return
def stuffDatabase(self):
"""
_stuffDatabase_
Fill the dbsbuffer with some files and blocks. We'll insert a total
of 5 files spanning two blocks. There will be a total of two datasets
inserted into the database.
All files will be already in GLOBAL and in_phedex
"""
myThread = threading.currentThread()
buffer3Factory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
insertWorkflow = buffer3Factory(classname = "InsertWorkflow")
insertWorkflow.execute("BogusRequestA", "BogusTask",
0, 0, 0, 0)
insertWorkflow.execute("BogusRequestB", "BogusTask",
0, 0, 0, 0)
checksums = {"adler32": "1234", "cksum": "5678"}
testFileA = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileA.setDatasetPath(self.testDatasetA)
testFileA.addRun(Run(2, *[45]))
testFileA.create()
testFileB = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileB.setDatasetPath(self.testDatasetA)
testFileB.addRun(Run(2, *[45]))
testFileB.create()
testFileC = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileC.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileC.setDatasetPath(self.testDatasetA)
testFileC.addRun(Run(2, *[45]))
testFileC.create()
self.testFilesA.append(testFileA)
self.testFilesA.append(testFileB)
self.testFilesA.append(testFileC)
testFileD = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileD.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileD.setDatasetPath(self.testDatasetB)
testFileD.addRun(Run(2, *[45]))
testFileD.create()
testFileE = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileE.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileE.setDatasetPath(self.testDatasetB)
testFileE.addRun(Run(2, *[45]))
testFileE.create()
self.testFilesB.append(testFileD)
self.testFilesB.append(testFileE)
uploadFactory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
datasetAction = uploadFactory(classname = "NewDataset")
createAction = uploadFactory(classname = "CreateBlocks")
datasetAction.execute(datasetPath = self.testDatasetA)
datasetAction.execute(datasetPath = self.testDatasetB)
self.blockAName = self.testDatasetA + "#" + makeUUID()
self.blockBName = self.testDatasetB + "#" + makeUUID()
newBlockA = DBSBlock(name = self.blockAName,
location = "srm-cms.cern.ch",
das = None, workflow = None)
newBlockA.setDataset(self.testDatasetA, 'data', 'VALID')
newBlockA.status = 'Closed'
newBlockB = DBSBlock(name = self.blockBName,
location = "srm-cms.cern.ch",
das = None, workflow = None)
newBlockB.setDataset(self.testDatasetB, 'data', 'VALID')
newBlockB.status = 'Closed'
createAction.execute(blocks = [newBlockA, newBlockB])
bufferFactory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
setBlock = bufferFactory(classname = "DBSBufferFiles.SetBlock")
setBlock.execute(testFileA["lfn"], self.blockAName)
setBlock.execute(testFileB["lfn"], self.blockAName)
setBlock.execute(testFileC["lfn"], self.blockAName)
setBlock.execute(testFileD["lfn"], self.blockBName)
setBlock.execute(testFileE["lfn"], self.blockBName)
fileStatus = bufferFactory(classname = "DBSBufferFiles.SetStatus")
fileStatus.execute(testFileA["lfn"], "GLOBAL")
fileStatus.execute(testFileB["lfn"], "GLOBAL")
fileStatus.execute(testFileC["lfn"], "GLOBAL")
fileStatus.execute(testFileD["lfn"], "GLOBAL")
fileStatus.execute(testFileE["lfn"], "GLOBAL")
phedexStatus = bufferFactory(classname = "DBSBufferFiles.SetPhEDExStatus")
phedexStatus.execute(testFileA["lfn"], 1)
phedexStatus.execute(testFileB["lfn"], 1)
phedexStatus.execute(testFileC["lfn"], 1)
phedexStatus.execute(testFileD["lfn"], 1)
phedexStatus.execute(testFileE["lfn"], 1)
associateWorkflow = buffer3Factory(classname = "DBSBufferFiles.AssociateWorkflowToFile")
associateWorkflow.execute(testFileA["lfn"], "BogusRequestA", "BogusTask")
associateWorkflow.execute(testFileB["lfn"], "BogusRequestA", "BogusTask")
associateWorkflow.execute(testFileC["lfn"], "BogusRequestA", "BogusTask")
associateWorkflow.execute(testFileD["lfn"], "BogusRequestB", "BogusTask")
associateWorkflow.execute(testFileE["lfn"], "BogusRequestB", "BogusTask")
# Make the desired subscriptions
insertSubAction = buffer3Factory(classname = "NewSubscription")
datasetA = DBSBufferDataset(path = self.testDatasetA)
datasetB = DBSBufferDataset(path = self.testDatasetB)
workload = WMWorkloadHelper()
workload.load(os.path.join(getTestBase(), 'WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl'))
insertSubAction.execute(datasetA.exists(), workload.getSubscriptionInformation()[self.testDatasetA])
insertSubAction.execute(datasetB.exists(), workload.getSubscriptionInformation()[self.testDatasetB])
return
def stuffDatabase(self):
"""
_stuffDatabase_
Fill the dbsbuffer with some files and blocks. We'll insert a total
of 5 files spanning two blocks. There will be a total of two datasets
inserted into the database.
All files will be already in GLOBAL and in_phedex
"""
myThread = threading.currentThread()
buffer3Factory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
insertWorkflow = buffer3Factory(classname="InsertWorkflow")
insertWorkflow.execute("BogusRequestA", "BogusTask",
0, 0, 0, 0)
insertWorkflow.execute("BogusRequestB", "BogusTask",
0, 0, 0, 0)
checksums = {"adler32": "1234", "cksum": "5678"}
testFileA = DBSBufferFile(lfn=makeUUID(), size=1024, events=10,
checksums=checksums,
locations=set(["srm-cms.cern.ch"]))
testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8",
appFam="RECO", psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFileA.setDatasetPath(self.testDatasetA)
testFileA.addRun(Run(2, *[45]))
testFileA.create()
testFileB = DBSBufferFile(lfn=makeUUID(), size=1024, events=10,
checksums=checksums,
locations=set(["srm-cms.cern.ch"]))
testFileB.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8",
appFam="RECO", psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFileB.setDatasetPath(self.testDatasetA)
testFileB.addRun(Run(2, *[45]))
testFileB.create()
testFileC = DBSBufferFile(lfn=makeUUID(), size=1024, events=10,
checksums=checksums,
locations=set(["srm-cms.cern.ch"]))
testFileC.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8",
appFam="RECO", psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFileC.setDatasetPath(self.testDatasetA)
testFileC.addRun(Run(2, *[45]))
testFileC.create()
self.testFilesA.append(testFileA)
self.testFilesA.append(testFileB)
self.testFilesA.append(testFileC)
testFileD = DBSBufferFile(lfn=makeUUID(), size=1024, events=10,
checksums=checksums,
locations=set(["srm-cms.cern.ch"]))
testFileD.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8",
appFam="RECO", psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFileD.setDatasetPath(self.testDatasetB)
testFileD.addRun(Run(2, *[45]))
testFileD.create()
testFileE = DBSBufferFile(lfn=makeUUID(), size=1024, events=10,
checksums=checksums,
locations=set(["srm-cms.cern.ch"]))
testFileE.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8",
appFam="RECO", psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFileE.setDatasetPath(self.testDatasetB)
testFileE.addRun(Run(2, *[45]))
testFileE.create()
self.testFilesB.append(testFileD)
self.testFilesB.append(testFileE)
uploadFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
datasetAction = uploadFactory(classname="NewDataset")
createAction = uploadFactory(classname="CreateBlocks")
datasetAction.execute(datasetPath=self.testDatasetA)
datasetAction.execute(datasetPath=self.testDatasetB)
self.blockAName = self.testDatasetA + "#" + makeUUID()
self.blockBName = self.testDatasetB + "#" + makeUUID()
newBlockA = DBSBufferBlock(name=self.blockAName,
location="srm-cms.cern.ch",
datasetpath=None)
newBlockA.setDataset(self.testDatasetA, 'data', 'VALID')
newBlockA.status = 'Closed'
newBlockB = DBSBufferBlock(name=self.blockBName,
location="srm-cms.cern.ch",
datasetpath=None)
newBlockB.setDataset(self.testDatasetB, 'data', 'VALID')
newBlockB.status = 'Closed'
createAction.execute(blocks=[newBlockA, newBlockB])
bufferFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
setBlock = bufferFactory(classname="DBSBufferFiles.SetBlock")
setBlock.execute(testFileA["lfn"], self.blockAName)
setBlock.execute(testFileB["lfn"], self.blockAName)
setBlock.execute(testFileC["lfn"], self.blockAName)
setBlock.execute(testFileD["lfn"], self.blockBName)
setBlock.execute(testFileE["lfn"], self.blockBName)
fileStatus = bufferFactory(classname="DBSBufferFiles.SetStatus")
fileStatus.execute(testFileA["lfn"], "GLOBAL")
fileStatus.execute(testFileB["lfn"], "GLOBAL")
fileStatus.execute(testFileC["lfn"], "GLOBAL")
fileStatus.execute(testFileD["lfn"], "GLOBAL")
fileStatus.execute(testFileE["lfn"], "GLOBAL")
phedexStatus = bufferFactory(classname="DBSBufferFiles.SetPhEDExStatus")
phedexStatus.execute(testFileA["lfn"], 1)
phedexStatus.execute(testFileB["lfn"], 1)
phedexStatus.execute(testFileC["lfn"], 1)
phedexStatus.execute(testFileD["lfn"], 1)
phedexStatus.execute(testFileE["lfn"], 1)
associateWorkflow = buffer3Factory(classname="DBSBufferFiles.AssociateWorkflowToFile")
associateWorkflow.execute(testFileA["lfn"], "BogusRequestA", "BogusTask")
associateWorkflow.execute(testFileB["lfn"], "BogusRequestA", "BogusTask")
associateWorkflow.execute(testFileC["lfn"], "BogusRequestA", "BogusTask")
associateWorkflow.execute(testFileD["lfn"], "BogusRequestB", "BogusTask")
associateWorkflow.execute(testFileE["lfn"], "BogusRequestB", "BogusTask")
# Make the desired subscriptions
insertSubAction = buffer3Factory(classname="NewSubscription")
datasetA = DBSBufferDataset(path=self.testDatasetA)
datasetB = DBSBufferDataset(path=self.testDatasetB)
workload = WMWorkloadHelper()
workload.load(os.path.join(getTestBase(), 'WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl'))
insertSubAction.execute(datasetA.exists(), workload.getSubscriptionInformation()[self.testDatasetA])
insertSubAction.execute(datasetB.exists(), workload.getSubscriptionInformation()[self.testDatasetB])
return
def algorithm(self, parameters):
"""
_algorithm_
Poll the database for datasets and subscribe them.
"""
myThread = threading.currentThread()
myThread.transaction.begin()
# Check for completely unsubscribed datasets
unsubscribedDatasets = self.getUnsubscribed.execute(conn = myThread.transaction.conn,
transaction = True)
if self.safeMode:
partiallySubscribedDatasets = self.getPartiallySubscribed.execute(conn = myThread.transaction.conn,
transaction = True)
unsubscribedDatasets.extend(partiallySubscribedDatasets)
partiallySubscribedSet = set()
for entry in partiallySubscribedDatasets:
partiallySubscribedSet.add(entry["path"])
# Map the datasets to their specs
specDatasetMap = {}
for unsubscribedDataset in unsubscribedDatasets:
datasetPath = unsubscribedDataset["path"]
workflow = unsubscribedDataset["workflow"]
spec = unsubscribedDataset["spec"]
if datasetPath not in specDatasetMap:
specDatasetMap[datasetPath] = []
specDatasetMap[datasetPath].append({"workflow" : workflow, "spec" : spec})
specCache = {}
siteMap = {}
# Distribute the subscriptions by site, type and priority
# This is to make as few subscriptions as possible
# Site map values are dictionaries where the keys are tuples (Prio, Custodial, AutoApprove, Move)
# Where Custodial is boolean, Prio is in ["Low", "Normal", "High"], AutoApprove is boolean and Move is boolean
for dataset in specDatasetMap:
# Aggregate all the different subscription configurations
subInfo = {}
for entry in specDatasetMap[dataset]:
if not entry["spec"]:
# Can't use this spec, there isn't one
continue
# Load spec if not in the cache
if entry["spec"] not in specCache:
helper = WMWorkloadHelper()
try:
helper.load(entry["spec"])
specCache[entry["spec"]] = helper
except Exception:
#Couldn't load it , alert and carry on
msg = "Couldn't load spec: %s" % entry["spec"]
logging.error(msg)
self.sendAlert(7, msg = msg)
continue
#If we are running in safe mode, we need to know if the workflow is ready
# We have the spec, get the info
helper = specCache[entry["spec"]]
workflowSubInfo = helper.getSubscriptionInformation()
datasetSubInfo = workflowSubInfo.get(dataset, None)
if datasetSubInfo and subInfo:
subInfo["CustodialSites"] = extendWithoutDups(subInfo["CustodialSites"], datasetSubInfo["CustodialSites"])
subInfo["NonCustodialSites"] = extendWithoutDups(subInfo["NonCustodialSites"], datasetSubInfo["NonCustodialSites"])
subInfo["AutoApproveSites"] = extendWithoutDups(subInfo["AutoApproveSites"], datasetSubInfo["AutoApproveSites"])
subInfo["Priority"] = solvePrioConflicts(subInfo["Priority"], datasetSubInfo["Priority"])
elif datasetSubInfo:
subInfo = datasetSubInfo
# We now have aggregated subscription information for this dataset in subInfo
# Distribute it by site
if not subInfo:
#Nothing to do, log and continue
msg = "No subscriptions configured for dataset %s" % dataset
logging.warning(msg)
self.markSubscribed.execute(dataset, subscribed = self.terminalSubscriptionState,
conn = myThread.transaction.conn,
transaction = True)
continue
# Make sure that a site is not configured both as non custodial and custodial
# Non-custodial is believed to be the right choice
subInfo["CustodialSites"] = list(set(subInfo["CustodialSites"]) - set(subInfo["NonCustodialSites"]))
for site in subInfo["CustodialSites"]:
if site not in siteMap:
siteMap[site] = {}
if self.safeMode and dataset not in partiallySubscribedSet:
tupleKey = (subInfo["Priority"], True, False, False)
else:
tupleKey = (subInfo["Priority"], True, False, True)
if tupleKey not in siteMap[site]:
siteMap[site][tupleKey] = []
siteMap[site][tupleKey].append(dataset)
# If we are in safe mode and this is a partially subscribed dataset,
# then the non-custodial were done in a previous cycle
if self.safeMode and dataset in partiallySubscribedSet:
self.markSubscribed.execute(dataset, subscribed = self.terminalSubscriptionState,
conn = myThread.transaction.conn,
transaction = True)
continue
for site in subInfo["NonCustodialSites"]:
if site not in siteMap:
siteMap[site] = {}
autoApprove = False
if site in subInfo["AutoApproveSites"]:
autoApprove = True
tupleKey = (subInfo["Priority"], False, autoApprove)
if tupleKey not in siteMap[site]:
siteMap[site][tupleKey] = []
siteMap[site][tupleKey].append(dataset)
self.markSubscribed.execute(dataset, subscribed = 1,
conn = myThread.transaction.conn,
transaction = True)
# Actually request the subscriptions
for site in siteMap:
# Check that the site is valid
if site not in self.cmsToPhedexMap:
msg = "Site %s doesn't appear to be valid to PhEDEx" % site
logging.error(msg)
self.sendAlert(7, msg = msg)
continue
for subscriptionFlavor in siteMap[site]:
datasets = siteMap[site][subscriptionFlavor]
# Check that the site is valid
if "MSS" in self.cmsToPhedexMap[site]:
phedexNode = self.cmsToPhedexMap[site]["MSS"]
else:
phedexNode = self.cmsToPhedexMap[site]["Disk"]
logging.info("Subscribing %s to %s" % (datasets, site))
options = {"custodial" : "n", "requestOnly" : "y",
"priority" : subscriptionFlavor[0].lower(),
"move" : "n"}
if subscriptionFlavor[1]:
options["custodial"] = "y"
if subscriptionFlavor[3]:
options["move"] = "y"
if subscriptionFlavor[2]:
options["requestOnly"] = "n"
newSubscription = PhEDExSubscription(datasets, phedexNode, self.group,
**options)
xmlData = XMLDrop.makePhEDExXMLForDatasets(self.dbsUrl,
newSubscription.getDatasetPaths())
logging.debug(str(xmlData))
self.phedex.subscribe(newSubscription, xmlData)
myThread.transaction.commit()
return
def stuffDatabase(self, tier0Mode = False):
"""
_stuffDatabase_
Fill the dbsbuffer with some files and blocks. We'll insert a total
of 5 files spanning two blocks. There will be a total of two datasets
inserted into the database.
All files will be already in GLOBAL and in_phedex
"""
myThread = threading.currentThread()
buffer3Factory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
insertWorkflow = buffer3Factory(classname = "InsertWorkflow")
insertWorkflow.execute("BogusRequestA", "BogusTask",
0, 0, 0, 0)
insertWorkflow.execute("BogusRequestB", "BogusTask",
0, 0, 0, 0)
checksums = {"adler32": "1234", "cksum": "5678"}
testFileA = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileA.setDatasetPath(self.testDatasetA)
testFileA.addRun(Run(2, *[45]))
testFileA.create()
testFileB = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileB.setDatasetPath(self.testDatasetA)
testFileB.addRun(Run(2, *[45]))
testFileB.create()
testFileC = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileC.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileC.setDatasetPath(self.testDatasetA)
testFileC.addRun(Run(2, *[45]))
testFileC.create()
self.testFilesA.append(testFileA)
self.testFilesA.append(testFileB)
self.testFilesA.append(testFileC)
testFileD = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileD.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileD.setDatasetPath(self.testDatasetB)
testFileD.addRun(Run(2, *[45]))
testFileD.create()
testFileE = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10,
checksums = checksums,
locations = set(["srm-cms.cern.ch"]))
testFileE.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileE.setDatasetPath(self.testDatasetB)
testFileE.addRun(Run(2, *[45]))
testFileE.create()
self.testFilesB.append(testFileD)
self.testFilesB.append(testFileE)
uploadFactory = DAOFactory(package = "WMComponent.DBSUpload.Database",
logger = myThread.logger,
dbinterface = myThread.dbi)
createBlock = uploadFactory(classname = "SetBlockStatus")
self.blockAName = self.testDatasetA + "#" + makeUUID()
self.blockBName = self.testDatasetB + "#" + makeUUID()
createBlock.execute(block = self.blockAName, locations = ["srm-cms.cern.ch"], open_status = 'Closed')
createBlock.execute(block = self.blockBName, locations = ["srm-cms.cern.ch"], open_status = 'Closed')
bufferFactory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
setBlock = bufferFactory(classname = "DBSBufferFiles.SetBlock")
setBlock.execute(testFileA["lfn"], self.blockAName)
setBlock.execute(testFileB["lfn"], self.blockAName)
setBlock.execute(testFileC["lfn"], self.blockAName)
setBlock.execute(testFileD["lfn"], self.blockBName)
setBlock.execute(testFileE["lfn"], self.blockBName)
fileStatus = bufferFactory(classname = "DBSBufferFiles.SetStatus")
fileStatus.execute(testFileA["lfn"], "GLOBAL")
fileStatus.execute(testFileB["lfn"], "GLOBAL")
fileStatus.execute(testFileC["lfn"], "GLOBAL")
fileStatus.execute(testFileD["lfn"], "GLOBAL")
fileStatus.execute(testFileE["lfn"], "GLOBAL")
phedexStatus = bufferFactory(classname = "DBSBufferFiles.SetPhEDExStatus")
phedexStatus.execute(testFileA["lfn"], 1)
phedexStatus.execute(testFileB["lfn"], 1)
phedexStatus.execute(testFileC["lfn"], 1)
phedexStatus.execute(testFileD["lfn"], 1)
phedexStatus.execute(testFileE["lfn"], 1)
associateWorkflow = buffer3Factory(classname = "DBSBufferFiles.AssociateWorkflowToFile")
associateWorkflow.execute(testFileA["lfn"], "BogusRequestA", "BogusTask")
associateWorkflow.execute(testFileB["lfn"], "BogusRequestA", "BogusTask")
associateWorkflow.execute(testFileC["lfn"], "BogusRequestA", "BogusTask")
associateWorkflow.execute(testFileD["lfn"], "BogusRequestB", "BogusTask")
associateWorkflow.execute(testFileE["lfn"], "BogusRequestB", "BogusTask")
# Make the desired subscriptions
insertSubAction = buffer3Factory(classname = "NewSubscription")
datasetA = DBSBufferDataset(path = self.testDatasetA)
datasetB = DBSBufferDataset(path = self.testDatasetB)
workload = WMWorkloadHelper()
workload.load(os.path.join(getTestBase(), 'WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl'))
if tier0Mode:
# Override the settings
workload.setSubscriptionInformation(custodialSites = ["T0_CH_CERN", "T1_US_FNAL"],
nonCustodialSites = ["T3_CO_Uniandes"],
priority = "Normal", custodialSubType = "Replica",
autoApproveSites = ["T0_CH_CERN"],
dataTier = "RECO")
workload.setSubscriptionInformation(custodialSites = ["T0_CH_CERN", "T1_UK_RAL"],
nonCustodialSites = [],
autoApproveSites = [],
priority = "High", custodialSubType = "Replica",
dataTier = "RAW")
insertSubAction.execute(datasetA.exists(), workload.getSubscriptionInformation()[self.testDatasetA])
insertSubAction.execute(datasetB.exists(), workload.getSubscriptionInformation()[self.testDatasetB])
return
def algorithm(self, parameters):
"""
_algorithm_
Poll the database for datasets and subscribe them.
"""
myThread = threading.currentThread()
myThread.transaction.begin()
# Check for completely unsubscribed datasets
unsubscribedDatasets = self.getUnsubscribed.execute(
conn=myThread.transaction.conn, transaction=True)
if self.safeMode:
partiallySubscribedDatasets = self.getPartiallySubscribed.execute(
conn=myThread.transaction.conn, transaction=True)
unsubscribedDatasets.extend(partiallySubscribedDatasets)
partiallySubscribedSet = set()
for entry in partiallySubscribedDatasets:
partiallySubscribedSet.add(entry["path"])
# Map the datasets to their specs
specDatasetMap = {}
for unsubscribedDataset in unsubscribedDatasets:
datasetPath = unsubscribedDataset["path"]
workflow = unsubscribedDataset["workflow"]
spec = unsubscribedDataset["spec"]
if datasetPath not in specDatasetMap:
specDatasetMap[datasetPath] = []
specDatasetMap[datasetPath].append({
"workflow": workflow,
"spec": spec
})
specCache = {}
siteMap = {}
# Distribute the subscriptions by site, type and priority
# This is to make as few subscriptions as possible
# Site map values are dictionaries where the keys are tuples (Prio, Custodial, AutoApprove, Move)
# Where Custodial is boolean, Prio is in ["Low", "Normal", "High"], AutoApprove is boolean and Move is boolean
for dataset in specDatasetMap:
# Aggregate all the different subscription configurations
subInfo = {}
for entry in specDatasetMap[dataset]:
if not entry["spec"]:
# Can't use this spec, there isn't one
continue
# Load spec if not in the cache
if entry["spec"] not in specCache:
helper = WMWorkloadHelper()
try:
helper.load(entry["spec"])
specCache[entry["spec"]] = helper
except Exception:
#Couldn't load it , alert and carry on
msg = "Couldn't load spec: %s" % entry["spec"]
logging.error(msg)
self.sendAlert(7, msg=msg)
continue
#If we are running in safe mode, we need to know if the workflow is ready
# We have the spec, get the info
helper = specCache[entry["spec"]]
workflowSubInfo = helper.getSubscriptionInformation()
datasetSubInfo = workflowSubInfo.get(dataset, None)
if datasetSubInfo and subInfo:
subInfo["CustodialSites"] = extendWithoutDups(
subInfo["CustodialSites"],
datasetSubInfo["CustodialSites"])
subInfo["NonCustodialSites"] = extendWithoutDups(
subInfo["NonCustodialSites"],
datasetSubInfo["NonCustodialSites"])
subInfo["AutoApproveSites"] = extendWithoutDups(
subInfo["AutoApproveSites"],
datasetSubInfo["AutoApproveSites"])
subInfo["Priority"] = solvePrioConflicts(
subInfo["Priority"], datasetSubInfo["Priority"])
elif datasetSubInfo:
subInfo = datasetSubInfo
# We now have aggregated subscription information for this dataset in subInfo
# Distribute it by site
if not subInfo:
#Nothing to do, log and continue
msg = "No subscriptions configured for dataset %s" % dataset
logging.warning(msg)
self.markSubscribed.execute(
dataset,
subscribed=self.terminalSubscriptionState,
conn=myThread.transaction.conn,
transaction=True)
continue
# Make sure that a site is not configured both as non custodial and custodial
# Non-custodial is believed to be the right choice
subInfo["CustodialSites"] = list(
set(subInfo["CustodialSites"]) -
set(subInfo["NonCustodialSites"]))
for site in subInfo["CustodialSites"]:
if site not in siteMap:
siteMap[site] = {}
autoApprove = False
if site in subInfo["AutoApproveSites"]:
autoApprove = True
if self.safeMode and dataset not in partiallySubscribedSet:
tupleKey = (subInfo["Priority"], True, autoApprove, False)
else:
tupleKey = (subInfo["Priority"], True, autoApprove, True)
if tupleKey not in siteMap[site]:
siteMap[site][tupleKey] = []
# Subscriptions are sorted by options, defined by tupleKey
# The tuple key has 3 or 4 entries in this order
# Priority, Custodial, Auto approve, Move (True) or Replica (False)
siteMap[site][tupleKey].append(dataset)
# If we are in safe mode and this is a partially subscribed dataset,
# then the non-custodial were done in a previous cycle
if self.safeMode and dataset in partiallySubscribedSet:
self.markSubscribed.execute(
dataset,
subscribed=self.terminalSubscriptionState,
conn=myThread.transaction.conn,
transaction=True)
continue
for site in subInfo["NonCustodialSites"]:
if site not in siteMap:
siteMap[site] = {}
autoApprove = False
if site in subInfo["AutoApproveSites"]:
autoApprove = True
# Non-custodial is never move, so this tuple has only 3 entries
# TODO: Change tuples to frozensets for clarity
tupleKey = (subInfo["Priority"], False, autoApprove)
if tupleKey not in siteMap[site]:
siteMap[site][tupleKey] = []
siteMap[site][tupleKey].append(dataset)
self.markSubscribed.execute(dataset,
subscribed=1,
conn=myThread.transaction.conn,
transaction=True)
# Actually request the subscriptions
for site in siteMap:
# Check that the site is valid
if site not in self.cmsToPhedexMap:
msg = "Site %s doesn't appear to be valid to PhEDEx" % site
logging.error(msg)
self.sendAlert(7, msg=msg)
continue
for subscriptionFlavor in siteMap[site]:
datasets = siteMap[site][subscriptionFlavor]
# Check that the site is valid
isMSS = False
if "MSS" in self.cmsToPhedexMap[site]:
isMSS = True
phedexNode = self.cmsToPhedexMap[site]["MSS"]
else:
phedexNode = self.cmsToPhedexMap[site]["Disk"]
logging.info("Subscribing %s to %s" % (datasets, site))
options = {
"custodial": "n",
"requestOnly": "y",
"priority": subscriptionFlavor[0].lower(),
"move": "n"
}
if subscriptionFlavor[1] and isMSS:
# Custodial subscriptions are only allowed in MSS nodes
# If custodial is requested on Non-MSS it fallsback to a non-custodial subscription
options["custodial"] = "y"
if subscriptionFlavor[3] and not self.replicaOnly:
options["move"] = "y"
if subscriptionFlavor[2]:
options["requestOnly"] = "n"
logging.info(
"Request options: Custodial - %s, Move - %s, Request Only - %s"
% (options["custodial"].upper(), options["move"].upper(),
options["requestOnly"].upper()))
newSubscription = PhEDExSubscription(datasets, phedexNode,
self.group, **options)
xmlData = XMLDrop.makePhEDExXMLForDatasets(
self.dbsUrl, newSubscription.getDatasetPaths())
logging.debug(str(xmlData))
self.phedex.subscribe(newSubscription, xmlData)
myThread.transaction.commit()
return
