def testSubscription(self):
"""
_testSubscription_
Verify that the subscription API works.
"""
datasetA = "/%s/WMCorePhEDExTest/RAW" % makeUUID()
datasetB = "/%s/WMCorePhEDExTest/RECO" % makeUUID()
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetA)
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetB)
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
testSub = PhEDExSubscription([datasetA, datasetB], "T1_UK_RAL_MSS",
"Saturn")
xmlData = XMLDrop.makePhEDExXMLForDatasets(self.dbsTestUrl,
testSub.getDatasetPaths())
result = self.phedexApi.subscribe(testSub, xmlData)
requestIDs = result["phedex"]["request_created"]
self.assertEqual(len(requestIDs), 1,
"Error: Wrong number of request IDs")
self.assertTrue(requestIDs[0].has_key("id"),
"Error: Missing request ID")
return
def subscribeBlocks(self):
"""
_subscribeBlocks_
Poll the database and subscribe blocks not yet subscribed.
"""
logging.info("Starting subscribeBlocks method")
unsubBlocks = self.getUnsubscribedBlocks.execute()
# now organize those by location in order to minimize phedex requests
# also remove blocks that this component is meant to skip
unsubBlocks = self.organizeBlocksByLocation(unsubBlocks)
for location, blockDict in unsubBlocks.items():
phedexSub = PhEDExSubscription(blockDict.keys(),
location,
self.phedexGroup,
blocks=blockDict,
level="block",
priority="normal",
move="n",
custodial="n",
request_only="n",
comments="WMAgent production site")
try:
res = self.phedex.subscribe(phedexSub)
transferId = res['phedex']['request_created'][0]['id']
logging.info(
"Subscribed %d blocks for %d datasets, to location: %s, under request ID: %s",
len(phedexSub.getBlocks()),
len(phedexSub.getDatasetPaths()), phedexSub.getNodes(),
transferId)
except HTTPException as ex:
logging.error(
"PhEDEx block subscription failed with HTTPException: %s %s",
ex.status, ex.result)
logging.error("The subscription object was: %s",
str(phedexSub))
except Exception as ex:
logging.exception(
"PhEDEx block subscription failed with Exception: %s",
str(ex))
else:
binds = []
for blockname in phedexSub.getBlocks():
binds.append({
'RULE_ID': str(transferId),
'BLOCKNAME': blockname
})
self.setBlockRules.execute(binds)
return
def testGetSubscriptionMapping(self):
"""
_testGetSubscriptionMapping_
Verify that the subscription mapping API works correctly.
"""
testDataset = "/%s/WMCorePhEDExTest/RECO" % makeUUID()
blockA = "%s#%s" % (testDataset, makeUUID())
blockB = "%s#%s" % (testDataset, makeUUID())
injectionSpec = XMLDrop.XMLInjectionSpec(self.dbsTestUrl)
datasetSpec = injectionSpec.getDataset(testDataset)
datasetSpec.getFileblock(blockA, 'y')
datasetSpec.getFileblock(blockB, 'y')
blockSpec = injectionSpec.save()
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", blockSpec)
# Create a dataset level subscription to a node
testDatasetSub = PhEDExSubscription([testDataset],
"T1_UK_RAL_MSS",
"Saturn",
requestOnly="n")
datasetSpec = XMLDrop.makePhEDExXMLForDatasets(
self.dbsTestUrl, testDatasetSub.getDatasetPaths())
self.phedexApi.subscribe(testDatasetSub, datasetSpec)
# Create a block level subscrtion to a different node
testBlockSub = PhEDExSubscription([testDataset],
"T1_DE_KIT_MSS",
"Saturn",
level="block",
requestOnly="n")
self.phedexApi.subscribe(testBlockSub, blockSpec)
subs = self.phedexApi.getSubscriptionMapping(testDataset)
self.assertEqual(subs[testDataset], set(["T1_UK_RAL_MSS"]),
"Error: Dataset subscription is wrong.")
subs = self.phedexApi.getSubscriptionMapping(blockA)
self.assertEqual(
len(subs[blockA]), 2,
"Error: Wrong number of nodes in block subscription.")
self.assertTrue("T1_UK_RAL_MSS" in subs[blockA],
"Error: RAL missing from block sub.")
self.assertTrue("T1_DE_KIT_MSS" in subs[blockA],
"Error: KIT missing from block sub.")
return
def algorithm(self, parameters):
"""
_algorithm_
Poll the database for datasets and subscribe them.
"""
myThread = threading.currentThread()
myThread.transaction.begin()
# Check for completely unsubscribed datasets
unsubscribedDatasets = self.getUnsubscribed.execute(
conn=myThread.transaction.conn, transaction=True)
if self.safeMode:
partiallySubscribedDatasets = self.getPartiallySubscribed.execute(
conn=myThread.transaction.conn, transaction=True)
unsubscribedDatasets.extend(partiallySubscribedDatasets)
partiallySubscribedSet = set()
for entry in partiallySubscribedDatasets:
partiallySubscribedSet.add(entry["path"])
# Map the datasets to their specs
specDatasetMap = {}
for unsubscribedDataset in unsubscribedDatasets:
datasetPath = unsubscribedDataset["path"]
workflow = unsubscribedDataset["workflow"]
spec = unsubscribedDataset["spec"]
if datasetPath not in specDatasetMap:
specDatasetMap[datasetPath] = []
specDatasetMap[datasetPath].append({
"workflow": workflow,
"spec": spec
})
specCache = {}
siteMap = {}
# Distribute the subscriptions by site, type and priority
# This is to make as few subscriptions as possible
# Site map values are dictionaries where the keys are tuples (Prio, Custodial, AutoApprove, Move)
# Where Custodial is boolean, Prio is in ["Low", "Normal", "High"], AutoApprove is boolean and Move is boolean
for dataset in specDatasetMap:
# Aggregate all the different subscription configurations
subInfo = {}
for entry in specDatasetMap[dataset]:
if not entry["spec"]:
# Can't use this spec, there isn't one
continue
# Load spec if not in the cache
if entry["spec"] not in specCache:
helper = WMWorkloadHelper()
try:
helper.load(entry["spec"])
specCache[entry["spec"]] = helper
except Exception:
#Couldn't load it , alert and carry on
msg = "Couldn't load spec: %s" % entry["spec"]
logging.error(msg)
self.sendAlert(7, msg=msg)
continue
#If we are running in safe mode, we need to know if the workflow is ready
# We have the spec, get the info
helper = specCache[entry["spec"]]
workflowSubInfo = helper.getSubscriptionInformation()
datasetSubInfo = workflowSubInfo.get(dataset, None)
if datasetSubInfo and subInfo:
subInfo["CustodialSites"] = extendWithoutDups(
subInfo["CustodialSites"],
datasetSubInfo["CustodialSites"])
subInfo["NonCustodialSites"] = extendWithoutDups(
subInfo["NonCustodialSites"],
datasetSubInfo["NonCustodialSites"])
subInfo["AutoApproveSites"] = extendWithoutDups(
subInfo["AutoApproveSites"],
datasetSubInfo["AutoApproveSites"])
subInfo["Priority"] = solvePrioConflicts(
subInfo["Priority"], datasetSubInfo["Priority"])
elif datasetSubInfo:
subInfo = datasetSubInfo
# We now have aggregated subscription information for this dataset in subInfo
# Distribute it by site
if not subInfo:
#Nothing to do, log and continue
msg = "No subscriptions configured for dataset %s" % dataset
logging.warning(msg)
self.markSubscribed.execute(
dataset,
subscribed=self.terminalSubscriptionState,
conn=myThread.transaction.conn,
transaction=True)
continue
# Make sure that a site is not configured both as non custodial and custodial
# Non-custodial is believed to be the right choice
subInfo["CustodialSites"] = list(
set(subInfo["CustodialSites"]) -
set(subInfo["NonCustodialSites"]))
for site in subInfo["CustodialSites"]:
if site not in siteMap:
siteMap[site] = {}
autoApprove = False
if site in subInfo["AutoApproveSites"]:
autoApprove = True
if self.safeMode and dataset not in partiallySubscribedSet:
tupleKey = (subInfo["Priority"], True, autoApprove, False)
else:
tupleKey = (subInfo["Priority"], True, autoApprove, True)
if tupleKey not in siteMap[site]:
siteMap[site][tupleKey] = []
# Subscriptions are sorted by options, defined by tupleKey
# The tuple key has 3 or 4 entries in this order
# Priority, Custodial, Auto approve, Move (True) or Replica (False)
siteMap[site][tupleKey].append(dataset)
# If we are in safe mode and this is a partially subscribed dataset,
# then the non-custodial were done in a previous cycle
if self.safeMode and dataset in partiallySubscribedSet:
self.markSubscribed.execute(
dataset,
subscribed=self.terminalSubscriptionState,
conn=myThread.transaction.conn,
transaction=True)
continue
for site in subInfo["NonCustodialSites"]:
if site not in siteMap:
siteMap[site] = {}
autoApprove = False
if site in subInfo["AutoApproveSites"]:
autoApprove = True
# Non-custodial is never move, so this tuple has only 3 entries
# TODO: Change tuples to frozensets for clarity
tupleKey = (subInfo["Priority"], False, autoApprove)
if tupleKey not in siteMap[site]:
siteMap[site][tupleKey] = []
siteMap[site][tupleKey].append(dataset)
self.markSubscribed.execute(dataset,
subscribed=1,
conn=myThread.transaction.conn,
transaction=True)
# Actually request the subscriptions
for site in siteMap:
# Check that the site is valid
if site not in self.cmsToPhedexMap:
msg = "Site %s doesn't appear to be valid to PhEDEx" % site
logging.error(msg)
self.sendAlert(7, msg=msg)
continue
for subscriptionFlavor in siteMap[site]:
datasets = siteMap[site][subscriptionFlavor]
# Check that the site is valid
isMSS = False
if "MSS" in self.cmsToPhedexMap[site]:
isMSS = True
phedexNode = self.cmsToPhedexMap[site]["MSS"]
else:
phedexNode = self.cmsToPhedexMap[site]["Disk"]
logging.info("Subscribing %s to %s" % (datasets, site))
options = {
"custodial": "n",
"requestOnly": "y",
"priority": subscriptionFlavor[0].lower(),
"move": "n"
}
if subscriptionFlavor[1] and isMSS:
# Custodial subscriptions are only allowed in MSS nodes
# If custodial is requested on Non-MSS it fallsback to a non-custodial subscription
options["custodial"] = "y"
if subscriptionFlavor[3] and not self.replicaOnly:
options["move"] = "y"
if subscriptionFlavor[2]:
options["requestOnly"] = "n"
logging.info(
"Request options: Custodial - %s, Move - %s, Request Only - %s"
% (options["custodial"].upper(), options["move"].upper(),
options["requestOnly"].upper()))
newSubscription = PhEDExSubscription(datasets, phedexNode,
self.group, **options)
xmlData = XMLDrop.makePhEDExXMLForDatasets(
self.dbsUrl, newSubscription.getDatasetPaths())
logging.debug(str(xmlData))
self.phedex.subscribe(newSubscription, xmlData)
myThread.transaction.commit()
return
