def testPoller(self):
"""
_testPoller_
Stuff the database and have the poller upload files to PhEDEx. Retrieve
replica information for the uploaded blocks and verify that all files
have been injected.
"""
return
self.stuffDatabase()
poller = PhEDExInjectorPoller(self.createConfig())
poller.setup(parameters=None)
poller.algorithm(parameters=None)
replicaInfo = self.retrieveReplicaInfoForBlock(self.blockAName)
goldenLFNs = []
for file in self.testFilesA:
goldenLFNs.append(file["lfn"])
for replicaFile in replicaInfo["file"]:
assert replicaFile["name"] in goldenLFNs, \
"Error: Extra file in replica block: %s" % replicaFile["name"]
goldenLFNs.remove(replicaFile["name"])
assert len(goldenLFNs) == 0, \
"Error: Files missing from PhEDEx replica: %s" % goldenLFNs
replicaInfo = self.retrieveReplicaInfoForBlock(self.blockBName)
goldenLFNs = []
for file in self.testFilesB:
goldenLFNs.append(file["lfn"])
for replicaFile in replicaInfo["file"]:
assert replicaFile["name"] in goldenLFNs, \
"Error: Extra file in replica block: %s" % replicaFile["name"]
goldenLFNs.remove(replicaFile["name"])
assert len(goldenLFNs) == 0, \
"Error: Files missing from PhEDEx replica: %s" % goldenLFNs
myThread = threading.currentThread()
daofactory = DAOFactory(package="WMComponent.DBSUpload.Database",
logger=myThread.logger,
dbinterface=myThread.dbi)
setBlock = daofactory(classname="SetBlockStatus")
setBlock.execute(self.blockAName,
locations=None,
open_status="InGlobalDBS")
poller.algorithm(parameters=None)
replicaInfo = self.retrieveReplicaInfoForBlock(self.blockAName)
assert replicaInfo["is_open"] == "n", \
"Error: block should be closed."
replicaInfo = self.retrieveReplicaInfoForBlock(self.blockBName)
assert replicaInfo["is_open"] == "y", \
"Error: block should be open."
return
def preInitialization(self):
pollInterval = self.config.PhEDExInjector.pollInterval
subInterval = self.config.PhEDExInjector.subscribeInterval
logging.info("Setting poll interval to %s seconds for inject",
pollInterval)
# retrieving the node mappings is fickle and can fail quite often
# hence only do it once (with retries) and pass it to the workers
phedex = PhEDEx({"endpoint": self.config.PhEDExInjector.phedexurl},
"json")
try:
nodeMappings = phedex.getNodeMap()
except Exception:
time.sleep(2)
try:
nodeMappings = phedex.getNodeMap()
except Exception:
time.sleep(4)
nodeMappings = phedex.getNodeMap()
myThread = threading.currentThread()
myThread.workerThreadManager.addWorker(
PhEDExInjectorPoller(self.config, phedex, nodeMappings),
pollInterval)
if getattr(self.config.PhEDExInjector, "subscribeDatasets", False):
# wait a bit for first poll cycle of PhEDExInjectorPoller to complete
# hopefully avoids intermingled logs (which can be confusing)
time.sleep(2)
logging.info("Setting poll interval to %s seconds for subscribe",
subInterval)
myThread.workerThreadManager.addWorker(
PhEDExInjectorSubscriber(self.config, phedex, nodeMappings),
subInterval)
def preInitialization(self):
pollInterval = self.config.PhEDExInjector.pollInterval
logging.info("Setting poll interval to %s seconds for inject",
pollInterval)
myThread = threading.currentThread()
myThread.workerThreadManager.addWorker(
PhEDExInjectorPoller(self.config), pollInterval)
def testPoller(self):
"""
_testPoller_
Stuff the database and have the poller upload files to PhEDEx. Retrieve
replica information for the uploaded blocks and verify that all files
have been injected.
"""
return
self.stuffDatabase()
poller = PhEDExInjectorPoller(self.createConfig())
poller.setup(parameters = None)
poller.algorithm(parameters = None)
replicaInfo = self.retrieveReplicaInfoForBlock(self.blockAName)
goldenLFNs = []
for file in self.testFilesA:
goldenLFNs.append(file["lfn"])
for replicaFile in replicaInfo["file"]:
assert replicaFile["name"] in goldenLFNs, \
"Error: Extra file in replica block: %s" % replicaFile["name"]
goldenLFNs.remove(replicaFile["name"])
assert len(goldenLFNs) == 0, \
"Error: Files missing from PhEDEx replica: %s" % goldenLFNs
replicaInfo = self.retrieveReplicaInfoForBlock(self.blockBName)
goldenLFNs = []
for file in self.testFilesB:
goldenLFNs.append(file["lfn"])
for replicaFile in replicaInfo["file"]:
assert replicaFile["name"] in goldenLFNs, \
"Error: Extra file in replica block: %s" % replicaFile["name"]
goldenLFNs.remove(replicaFile["name"])
assert len(goldenLFNs) == 0, \
"Error: Files missing from PhEDEx replica: %s" % goldenLFNs
myThread = threading.currentThread()
daofactory = DAOFactory(package = "WMComponent.DBSUpload.Database",
logger = myThread.logger,
dbinterface = myThread.dbi)
setBlock = daofactory(classname = "SetBlockStatus")
setBlock.execute(self.blockAName, locations = None,
open_status = "InGlobalDBS")
poller.algorithm(parameters = None)
replicaInfo = self.retrieveReplicaInfoForBlock(self.blockAName)
assert replicaInfo["is_open"] == "n", \
"Error: block should be closed."
replicaInfo = self.retrieveReplicaInfoForBlock(self.blockBName)
assert replicaInfo["is_open"] == "y", \
"Error: block should be open."
return
def preInitialization(self):
pollInterval = self.config.PhEDExInjector.pollInterval
subInterval = self.config.PhEDExInjector.subscribeInterval
logging.info("Setting poll interval to %s seconds for inject" %
pollInterval)
myThread = threading.currentThread()
myThread.workerThreadManager.addWorker(
PhEDExInjectorPoller(self.config), pollInterval)
if getattr(self.config.PhEDExInjector, "subscribeDatasets", False):
logging.info("Setting poll interval to %s seconds for subscribe" %
subInterval)
myThread.workerThreadManager.addWorker(
PhEDExInjectorSubscriber(self.config), subInterval)
def test_CustodialSiteA(self):
"""
_CustodialSiteA_
Check the custodialSite stuff by DAO, since I don't have a cert
First make sure we properly handle having no custodialSite
"""
self.stuffDatabase(custodialSite=None)
poller = PhEDExInjectorPoller(self.createConfig())
myThread = threading.currentThread()
daofactory = DAOFactory(package="WMComponent.PhEDExInjector.Database",
logger=myThread.logger,
dbinterface=myThread.dbi)
getUninjected = daofactory(classname="GetUninjectedFiles")
uninjectedFiles = getUninjected.execute()
self.assertEqual(uninjectedFiles.keys(), ['srm-cms.cern.ch'])
return
def testNormalModeSubscriptions(self):
"""
_testNormalModeSubscriptions_
Tests that we can make custodial/non-custodial subscriptions on
normal operation mode, this time we don't need WMBS for anything.
All is subscribed in one go.
Check that the requests are correct.
"""
self.stuffDatabase()
config = self.createConfig()
subscriber = PhEDExInjectorPoller(config)
subscriber.setup({})
subscriber.subscribeDatasets()
phedexInstance = subscriber.phedex
subscriptions = phedexInstance.subRequests
# Let's check /BogusPrimary/Run2012Z-PromptReco-v1/RECO
# According to the spec, this should be custodial at T1_US_FNAL
# Non-custodial at T1_UK_RAL and T3_CO_Uniandes
# Autoapproved in all sites
# Priority is normal
self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed")
subInfoA = subscriptions[self.testDatasetA]
self.assertEqual(len(subInfoA), 3, "Dataset A was not subscribed to all sites")
for subInfo in subInfoA:
site = subInfo["node"]
self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription")
if site == "T1_UK_RAL_MSS" or site == "T3_CO_Uniandes":
self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(
subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]
)
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"])
elif site == "T1_US_FNAL_MSS":
self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(
subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]
)
self.assertEqual(subInfo["move"], "y", "Wrong subscription type for dataset A at %s" % subInfo["node"])
else:
self.fail("Dataset A was subscribed to a wrong site %s" % site)
# Now check /BogusPrimary/CRUZET11-v1/RAW
# According to the spec, this is not custodial anywhere
# Non-custodial at T1_UK_RAL and T2_CH_CERN
# Request only at both sites and with high priority
self.assertTrue(self.testDatasetB in subscriptions, "Dataset B was not subscribed")
subInfoB = subscriptions[self.testDatasetB]
self.assertEqual(len(subInfoB), 2, "Dataset B was not subscribed to all sites")
for subInfo in subInfoB:
site = subInfo["node"]
self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription")
if site == "T1_UK_RAL_MSS" or site == "T2_CH_CERN":
self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset B at %s" % subInfo["node"])
self.assertEqual(
subInfo["request_only"], "y", "Wrong requestOnly for dataset B at %s" % subInfo["node"]
)
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset B at %s" % subInfo["node"])
else:
self.fail("Dataset B was subscribed to a wrong site %s" % site)
myThread = threading.currentThread()
result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset_subscription where subscribed = 1")[
0
].fetchall()
self.assertEqual(result[0][0], 5, "Not all datasets were marked as subscribed")
result = myThread.dbi.processData("SELECT site FROM dbsbuffer_dataset_subscription where subscribed = 0")[
0
].fetchall()
self.assertEqual(result[0][0], "T1_IT_CNAF", "A non-valid CMS site was subscribed")
# Reset and run again and make sure that no duplicate subscriptions are created
myThread.dbi.processData("UPDATE dbsbuffer_dataset_subscription SET subscribed = 0")
subscriber.subscribeDatasets()
self.assertEqual(len(subscriptions[self.testDatasetA]), 3)
self.assertEqual(len(subscriptions[self.testDatasetB]), 2)
return
def testPoller(self):
"""
_testPoller_
Stuff the database and have the poller upload files to PhEDEx. Retrieve
replica information for the uploaded blocks and verify that all files
have been injected. Also verify that files have been subscribed to MSS.
"""
self.stuffDatabase()
poller = PhEDExInjectorPoller(self.createConfig())
poller.setup(parameters = None)
poller.algorithm(parameters = None)
replicaInfo = self.retrieveReplicaInfoForBlock(self.blockAName)
goldenLFNs = []
for file in self.testFilesA:
goldenLFNs.append(file["lfn"])
for replicaFile in replicaInfo["file"]:
assert replicaFile["name"] in goldenLFNs, \
"Error: Extra file in replica block: %s" % replicaFile["name"]
goldenLFNs.remove(replicaFile["name"])
assert len(goldenLFNs) == 0, \
"Error: Files missing from PhEDEx replica: %s" % goldenLFNs
replicaInfo = self.retrieveReplicaInfoForBlock(self.blockBName)
goldenLFNs = []
for file in self.testFilesB:
goldenLFNs.append(file["lfn"])
for replicaFile in replicaInfo["file"]:
assert replicaFile["name"] in goldenLFNs, \
"Error: Extra file in replica block: %s" % replicaFile["name"]
goldenLFNs.remove(replicaFile["name"])
assert len(goldenLFNs) == 0, \
"Error: Files missing from PhEDEx replica: %s" % goldenLFNs
myThread = threading.currentThread()
daofactory = DAOFactory(package = "WMComponent.DBSUpload.Database",
logger = myThread.logger,
dbinterface = myThread.dbi)
setBlock = daofactory(classname = "SetBlockStatus")
setBlock.execute(self.blockAName, locations = None,
open_status = "InGlobalDBS")
poller.algorithm(parameters = None)
replicaInfo = self.retrieveReplicaInfoForBlock(self.blockAName)
assert replicaInfo["is_open"] == "n", \
"Error: block should be closed."
replicaInfo = self.retrieveReplicaInfoForBlock(self.blockBName)
assert replicaInfo["is_open"] == "y", \
"Error: block should be open."
subscriber = PhEDExInjectorSubscriber(self.createConfig())
subscriber.setup(parameters = None)
subscriber.algorithm(parameters = None)
subAResult = self.phedex.subscriptions(dataset = self.testDatasetA)
self.assertEqual(len(subAResult["phedex"]["dataset"]), 1,
"Error: Subscription was not made.")
datasetASub = subAResult["phedex"]["dataset"][0]
self.assertTrue(datasetASub["files"] == "3" and datasetASub["name"] == self.testDatasetA,
"Error: Metadata is incorrect for sub.")
self.assertEqual(datasetASub["subscription"][0]["node"], "T1_CH_CERN_MSS",
"Error: Node is wrong.")
return
def testNormalModeSubscriptions(self):
"""
_testNormalModeSubscriptions_
Tests that we can make custodial/non-custodial subscriptions on
normal operation mode, this time we don't need WMBS for anything.
All is subscribed in one go.
Check that the requests are correct.
"""
self.stuffDatabase()
config = self.createConfig()
subscriber = PhEDExInjectorPoller(config)
subscriber.setup({})
subscriber.subscribeDatasets()
phedexInstance = subscriber.phedex
subscriptions = phedexInstance.subRequests
# Let's check /BogusPrimary/Run2012Z-PromptReco-v1/RECO
# According to the spec, this should be custodial at T1_US_FNAL
# Non-custodial at T1_UK_RAL and T3_CO_Uniandes
# Autoapproved in all sites
# Priority is normal
self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed")
subInfoA = subscriptions[self.testDatasetA]
self.assertEqual(len(subInfoA), 3, "Dataset A was not subscribed to all sites")
for subInfo in subInfoA:
site = subInfo["node"]
self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription")
if site == "T1_UK_RAL_MSS" or site == "T3_CO_Uniandes":
self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "n",
"Wrong requestOnly for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"])
elif site == "T1_US_FNAL_MSS":
self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "n",
"Wrong requestOnly for dataset A at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "y", "Wrong subscription type for dataset A at %s" % subInfo["node"])
else:
self.fail("Dataset A was subscribed to a wrong site %s" % site)
# Now check /BogusPrimary/CRUZET11-v1/RAW
# According to the spec, this is not custodial anywhere
# Non-custodial at T1_UK_RAL and T2_CH_CERN
# Request only at both sites and with high priority
self.assertTrue(self.testDatasetB in subscriptions, "Dataset B was not subscribed")
subInfoB = subscriptions[self.testDatasetB]
self.assertEqual(len(subInfoB), 2, "Dataset B was not subscribed to all sites")
for subInfo in subInfoB:
site = subInfo["node"]
self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription")
if site == "T1_UK_RAL_MSS" or site == "T2_CH_CERN":
self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset B at %s" % subInfo["node"])
self.assertEqual(subInfo["request_only"], "y",
"Wrong requestOnly for dataset B at %s" % subInfo["node"])
self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset B at %s" % subInfo["node"])
else:
self.fail("Dataset B was subscribed to a wrong site %s" % site)
myThread = threading.currentThread()
result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset_subscription where subscribed = 1")[
0].fetchall()
self.assertEqual(result[0][0], 5, "Not all datasets were marked as subscribed")
result = myThread.dbi.processData("SELECT site FROM dbsbuffer_dataset_subscription where subscribed = 0")[
0].fetchall()
self.assertEqual(result[0][0], "T1_IT_CNAF", "A non-valid CMS site was subscribed")
# Reset and run again and make sure that no duplicate subscriptions are created
myThread.dbi.processData("UPDATE dbsbuffer_dataset_subscription SET subscribed = 0")
subscriber.subscribeDatasets()
self.assertEqual(len(subscriptions[self.testDatasetA]), 3)
self.assertEqual(len(subscriptions[self.testDatasetB]), 2)
return