def testSubscription(self):
"""
_testSubscription_
Verify that the subscription API works.
"""
datasetA = "/%s/WMCorePhEDExTest/RAW" % makeUUID()
datasetB = "/%s/WMCorePhEDExTest/RECO" % makeUUID()
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetA)
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetB)
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
testSub = PhEDExSubscription([datasetA, datasetB], "T1_UK_RAL_MSS",
"Saturn")
xmlData = XMLDrop.makePhEDExXMLForDatasets(self.dbsTestUrl,
testSub.getDatasetPaths())
result = self.phedexApi.subscribe(testSub, xmlData)
requestIDs = result["phedex"]["request_created"]
self.assertEqual(len(requestIDs), 1,
"Error: Wrong number of request IDs")
self.assertTrue(requestIDs[0].has_key("id"),
"Error: Missing request ID")
return
def testSubscription(self):
"""
_testSubscription_
Verify that the subscription API works.
"""
datasetA = "/%s/WMCorePhEDExTest/RAW" % makeUUID()
datasetB = "/%s/WMCorePhEDExTest/RECO" % makeUUID()
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetA)
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
xmlData = XMLDrop.makePhEDExDrop(self.dbsTestUrl, datasetB)
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", xmlData)
testSub = PhEDExSubscription([datasetA, datasetB], "T1_UK_RAL_MSS",
"Saturn")
xmlData = XMLDrop.makePhEDExXMLForDatasets(self.dbsTestUrl,
testSub.getDatasetPaths())
result = self.phedexApi.subscribe(testSub, xmlData)
requestIDs = result["phedex"]["request_created"]
self.assertEqual(len(requestIDs), 1,
"Error: Wrong number of request IDs")
self.assertTrue(requestIDs[0].has_key("id"),
"Error: Missing request ID")
return
def subscribeTier0Blocks(self):
"""
_subscribeTier0Blocks_
Subscribe blocks to the Tier-0 where a replica subscription
already exists. All Tier-0 subscriptions are move, custodial
and autoapproved with high priority.
"""
myThread = threading.currentThread()
myThread.transaction.begin()
# Check for candidate blocks for subscription
blocksToSubscribe = self.getUnsubscribedBlocks.execute(node = 'T0_CH_CERN',
conn = myThread.transaction.conn,
transaction = True)
if not blocksToSubscribe:
return
# For the blocks we don't really care about the subscription options
# We are subscribing all blocks with the same recipe.
subscriptionMap = {}
for subInfo in blocksToSubscribe:
dataset = subInfo['path']
if dataset not in subscriptionMap:
subscriptionMap[dataset] = []
subscriptionMap[dataset].append(subInfo['blockname'])
site = 'T0_CH_CERN'
custodial = 'y'
request_only = 'n'
move = 'y'
priority = 'High'
# Get the phedex node
phedexNode = self.cmsToPhedexMap[site]["MSS"]
logging.error("Subscribing %d blocks, from %d datasets to the Tier-0" % (len(subscriptionMap), sum([len(x) for x in subscriptionMap.values()])))
newSubscription = PhEDExSubscription(subscriptionMap.keys(),
phedexNode, self.group,
custodial = custodial,
request_only = request_only,
move = move,
priority = priority,
level = 'block',
blocks = subscriptionMap)
# TODO: Check for blocks already subscribed
try:
xmlData = XMLDrop.makePhEDExXMLForBlocks(self.dbsUrl,
newSubscription.getDatasetsAndBlocks())
logging.debug(str(xmlData))
self.phedex.subscribe(newSubscription, xmlData)
except Exception, ex:
logging.error("Something went wrong when communicating with PhEDEx, will try again later.")
logging.error("Exception: %s" % str(ex))
def testGetSubscriptionMapping(self):
"""
_testGetSubscriptionMapping_
Verify that the subscription mapping API works correctly.
"""
testDataset = "/%s/WMCorePhEDExTest/RECO" % makeUUID()
blockA = "%s#%s" % (testDataset, makeUUID())
blockB = "%s#%s" % (testDataset, makeUUID())
# NOTE: leaving it broken on purpose, we do NOT want to subscribe
# data via unit tests :-)
#injectionSpec = XMLDrop.XMLInjectionSpec(self.dbsTestUrl)
datasetSpec = injectionSpec.getDataset(testDataset)
datasetSpec.getFileblock(blockA, 'y')
datasetSpec.getFileblock(blockB, 'y')
blockSpec = injectionSpec.save()
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", blockSpec)
# Create a dataset level subscription to a node
testDatasetSub = PhEDExSubscription([testDataset],
"T1_UK_RAL_MSS",
"Saturn",
request_only="y")
self.phedexApi.subscribe(testDatasetSub)
# Create a block level subscrtion to a different node
testBlockSub = PhEDExSubscription([testDataset],
"T1_DE_KIT_MSS",
"Saturn",
level="block",
request_only="y")
self.phedexApi.subscribe(testBlockSub)
subs = self.phedexApi.getSubscriptionMapping(testDataset)
self.assertEqual(subs[testDataset], {"T1_UK_RAL_MSS"},
"Error: Dataset subscription is wrong.")
subs = self.phedexApi.getSubscriptionMapping(blockA)
self.assertEqual(
len(subs[blockA]), 2,
"Error: Wrong number of nodes in block subscription.")
self.assertTrue("T1_UK_RAL_MSS" in subs[blockA],
"Error: RAL missing from block sub.")
self.assertTrue("T1_DE_KIT_MSS" in subs[blockA],
"Error: KIT missing from block sub.")
return
def testGetSubscriptionMapping(self):
"""
_testGetSubscriptionMapping_
Verify that the subscription mapping API works correctly.
"""
testDataset = "/%s/WMCorePhEDExTest/RECO" % makeUUID()
blockA = "%s#%s" % (testDataset, makeUUID())
blockB = "%s#%s" % (testDataset, makeUUID())
injectionSpec = XMLDrop.XMLInjectionSpec(self.dbsTestUrl)
datasetSpec = injectionSpec.getDataset(testDataset)
datasetSpec.getFileblock(blockA, 'y')
datasetSpec.getFileblock(blockB, 'y')
blockSpec = injectionSpec.save()
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", blockSpec)
# Create a dataset level subscription to a node
testDatasetSub = PhEDExSubscription([testDataset],
"T1_UK_RAL_MSS",
"Saturn",
requestOnly="n")
datasetSpec = XMLDrop.makePhEDExXMLForDatasets(
self.dbsTestUrl, testDatasetSub.getDatasetPaths())
self.phedexApi.subscribe(testDatasetSub, datasetSpec)
# Create a block level subscrtion to a different node
testBlockSub = PhEDExSubscription([testDataset],
"T1_DE_KIT_MSS",
"Saturn",
level="block",
requestOnly="n")
self.phedexApi.subscribe(testBlockSub, blockSpec)
subs = self.phedexApi.getSubscriptionMapping(testDataset)
self.assertEqual(subs[testDataset], set(["T1_UK_RAL_MSS"]),
"Error: Dataset subscription is wrong.")
subs = self.phedexApi.getSubscriptionMapping(blockA)
self.assertEqual(
len(subs[blockA]), 2,
"Error: Wrong number of nodes in block subscription.")
self.assertTrue("T1_UK_RAL_MSS" in subs[blockA],
"Error: RAL missing from block sub.")
self.assertTrue("T1_DE_KIT_MSS" in subs[blockA],
"Error: KIT missing from block sub.")
return
def algorithm(self, parameters):
"""
_algorithm_
Poll the database for datasets and subscribe them to MSS.
"""
myThread = threading.currentThread()
myThread.transaction.begin()
if not self.seMap.has_key("MSS"):
return
unsubscribedDatasets = self.getUnsubscribed.execute(conn = myThread.transaction.conn,
transaction = True)
datasetMap = {}
for unsubscribedDataset in unsubscribedDatasets:
datasetPath = unsubscribedDataset["path"]
seName = unsubscribedDataset["se_name"]
if not self.seMap["MSS"].has_key(seName):
msg = "No MSS node for SE: %s" % seName
logging.error(msg)
self.sendAlert(7, msg = msg)
continue
if not datasetMap.has_key(self.seMap["MSS"][seName]):
datasetMap[self.seMap["MSS"][seName]] = []
datasetMap[self.seMap["MSS"][seName]].append(datasetPath)
self.markSubscribed.execute(datasetPath, conn = myThread.transaction.conn,
transaction = True)
for siteMSS in datasetMap.keys():
logging.info("Subscribing %s to %s" % (datasetMap[siteMSS],
siteMSS))
newSubscription = PhEDExSubscription(datasetMap[siteMSS], siteMSS, self.group,
custodial = "y", requestOnly = "y")
xmlData = XMLDrop.makePhEDExXMLForDatasets(self.dbsUrl,
newSubscription.getDatasetPaths())
print xmlData
self.phedex.subscribe(newSubscription, xmlData)
myThread.transaction.commit()
return
def testGetSubscriptionMapping(self):
"""
_testGetSubscriptionMapping_
Verify that the subscription mapping API works correctly.
"""
testDataset = "/%s/WMCorePhEDExTest/RECO" % makeUUID()
blockA = "%s#%s" % (testDataset, makeUUID())
blockB = "%s#%s" % (testDataset, makeUUID())
injectionSpec = XMLDrop.XMLInjectionSpec(self.dbsTestUrl)
datasetSpec = injectionSpec.getDataset(testDataset)
datasetSpec.getFileblock(blockA, 'y')
datasetSpec.getFileblock(blockB, 'y')
blockSpec = injectionSpec.save()
self.phedexApi.injectBlocks("T1_US_FNAL_MSS", blockSpec)
# Create a dataset level subscription to a node
testDatasetSub = PhEDExSubscription([testDataset], "T1_UK_RAL_MSS",
"Saturn", requestOnly = "n")
datasetSpec = XMLDrop.makePhEDExXMLForDatasets(self.dbsTestUrl,
testDatasetSub.getDatasetPaths())
self.phedexApi.subscribe(testDatasetSub, datasetSpec)
# Create a block level subscrtion to a different node
testBlockSub = PhEDExSubscription([testDataset], "T1_DE_KIT_MSS", "Saturn",
level = "block", requestOnly = "n")
self.phedexApi.subscribe(testBlockSub, blockSpec)
subs = self.phedexApi.getSubscriptionMapping(testDataset)
self.assertEqual(subs[testDataset], set(["T1_UK_RAL_MSS"]),
"Error: Dataset subscription is wrong.")
subs = self.phedexApi.getSubscriptionMapping(blockA)
self.assertEqual(len(subs[blockA]), 2,
"Error: Wrong number of nodes in block subscription.")
self.assertTrue("T1_UK_RAL_MSS" in subs[blockA],
"Error: RAL missing from block sub.")
self.assertTrue("T1_DE_KIT_MSS" in subs[blockA],
"Error: KIT missing from block sub.")
return
def transferRequest(self, req):
"Send request to Phedex and return status of request subscription"
datasets = req.get('datasets', [])
sites = req.get('sites', [])
if datasets and sites:
self.logger.debug("### creating subscription for: %s",
pformat(req))
subscription = PhEDExSubscription(datasets, sites,
self.msConfig['group'])
# TODO: implement how to get transfer id
tid = hashlib.md5(str(subscription)).hexdigest()
# TODO: when ready enable submit subscription step
# self.phedex.subscribe(subscription)
return tid
def add(self, requests):
"Add requests to task manager"
# loop over requests: for non-existing pid submit phedex subscription for existing one check their status
for request, rdict in requests.items():
if self.store.exists(request):
self.checkStatus(request)
else:
# if request does not exist in backend submit its subscription and add it to backend
rdict = requests[request]
self.store.add(request, rdict)
datasets = rdict.get('datasets')
sites = rdict.get('sites')
subscription = PhEDExSubscription(datasets, sites, self.group)
if self.verbose:
print("### add subscription", subscription)
def subscribeBlocks(self):
"""
_subscribeBlocks_
Poll the database and subscribe blocks not yet subscribed.
"""
logging.info("Starting subscribeBlocks method")
unsubBlocks = self.getUnsubscribedBlocks.execute()
# now organize those by location in order to minimize phedex requests
# also remove blocks that this component is meant to skip
unsubBlocks = self.organizeBlocksByLocation(unsubBlocks)
for location, blockDict in unsubBlocks.items():
phedexSub = PhEDExSubscription(blockDict.keys(),
location,
self.phedexGroup,
blocks=blockDict,
level="block",
priority="normal",
move="n",
custodial="n",
request_only="n",
comments="WMAgent production site")
try:
res = self.phedex.subscribe(phedexSub)
transferId = res['phedex']['request_created'][0]['id']
logging.info(
"Subscribed %d blocks for %d datasets, to location: %s, under request ID: %s",
len(phedexSub.getBlocks()),
len(phedexSub.getDatasetPaths()), phedexSub.getNodes(),
transferId)
except HTTPException as ex:
logging.error(
"PhEDEx block subscription failed with HTTPException: %s %s",
ex.status, ex.result)
logging.error("The subscription object was: %s",
str(phedexSub))
except Exception as ex:
logging.exception(
"PhEDEx block subscription failed with Exception: %s",
str(ex))
else:
binds = []
for blockname in phedexSub.getBlocks():
binds.append({
'RULE_ID': str(transferId),
'BLOCKNAME': blockname
})
self.setBlockRules.execute(binds)
return
def test_SubscriptionList(self):
"""
_SubscriptionList_
Whatever
"""
policy = SubscriptionList()
# what will you do with run ID.
row = [6, "/Cosmics/Test-CRAFTv8/RAW", 1, "T2_CH_CAF", 'high', 'y']
results = []
for i in range(6):
results.append(row)
results.append(
[7, "/Cosmics/Test-CRAFTv8/ALCARECO", 2, "FNAL", 'normal', 'y'])
results.append(
[8, "/Cosmics/Test-CRAFTv8/RECO", 1, "T2_CH_CAF", 'high', 'y'])
results.append(
[8, "/Cosmics/Test-CRAFTv8/RECO", 2, "FNAL", 'high', 'y'])
print policy.getSubscriptionList()
for row in results:
# make a tuple (dataset_path_id, dataset_path_name)
# make a tuple (node_id, node_name)
subscription = PhEDExSubscription((row[0], row[1]),
(row[2], row[3]), row[4], row[5])
policy.addSubscription(subscription)
i = 0
for sub in policy.getSubscriptionList():
i += 1
print "Subscription %s" % i
print sub.getDatasetPaths()
print sub.getNodes()
return
def subscribeDatasets(self):
"""
_subscribeDatasets_
Poll the database for datasets and subscribe them.
"""
logging.info("Starting subscribeDatasets method")
# Check for completely unsubscribed datasets
unsubscribedDatasets = self.getUnsubscribed.execute()
# Keep a list of subscriptions to tick as subscribed in the database
subscriptionsMade = []
# Create a list of subscriptions as defined by the PhEDEx data structures
subs = SubscriptionList()
# Create the subscription objects and add them to the list
# The list takes care of the sorting internally
for subInfo in unsubscribedDatasets:
site = subInfo['site']
if site not in self.phedexNodes[
'MSS'] and site not in self.phedexNodes['Disk']:
msg = "Site %s doesn't appear to be valid to PhEDEx, " % site
msg += "skipping subscription: %s" % subInfo['id']
logging.error(msg)
self.sendAlert(7, msg=msg)
continue
# Avoid custodial subscriptions to disk nodes
if site not in self.phedexNodes['MSS']:
subInfo['custodial'] = 'n'
# Avoid auto approval in T1 sites
elif site.startswith("T1"):
subInfo['request_only'] = 'y'
phedexSub = PhEDExSubscription(
subInfo['path'],
site,
subInfo['phedex_group'],
priority=subInfo['priority'],
move=subInfo['move'],
custodial=subInfo['custodial'],
request_only=subInfo['request_only'],
subscriptionId=subInfo['id'])
# Check if the subscription is a duplicate
if phedexSub.matchesExistingSubscription(self.phedex) or \
phedexSub.matchesExistingTransferRequest(self.phedex):
subscriptionsMade.append(subInfo['id'])
continue
# Add it to the list
subs.addSubscription(phedexSub)
# Compact the subscriptions
subs.compact()
for subscription in subs.getSubscriptionList():
xmlData = XMLDrop.makePhEDExXMLForDatasets(
self.dbsUrl, subscription.getDatasetPaths())
logging.debug("subscribeDatasets XMLData: %s", xmlData)
logging.info(
"Subscribing: %s to %s, with options: Move: %s, Custodial: %s, Request Only: %s",
subscription.getDatasetPaths(), subscription.getNodes(),
subscription.move, subscription.custodial,
subscription.request_only)
try:
self.phedex.subscribe(subscription, xmlData)
except HTTPException as ex:
logging.error(
"PhEDEx dataset subscribe failed with HTTPException: %s %s",
ex.status, ex.result)
except Exception as ex:
logging.error(
"PhEDEx dataset subscribe failed with Exception: %s",
str(ex))
logging.debug("Traceback: %s", str(traceback.format_exc()))
else:
subscriptionsMade.extend(subscription.getSubscriptionIds())
# Register the result in DBSBuffer
if subscriptionsMade:
self.markSubscribed.execute(subscriptionsMade)
return
def subscribeDatasets(self):
"""
_subscribeDatasets_
Poll the database for datasets and subscribe them.
"""
myThread = threading.currentThread()
myThread.transaction.begin()
# Check for completely unsubscribed datasets
unsubscribedDatasets = self.getUnsubscribed.execute(conn = myThread.transaction.conn,
transaction = True)
# Keep a list of subscriptions to tick as subscribed in the database
subscriptionsMade = []
# Create a list of subscriptions as defined by the PhEDEx data structures
subs = SubscriptionList()
# Create the subscription objects and add them to the list
# The list takes care of the sorting internally
for subInfo in unsubscribedDatasets:
site = subInfo['site']
if site not in self.phedexNodes['MSS'] and site not in self.phedexNodes['Disk']:
if site not in self.cmsToPhedexMap:
msg = "Site %s doesn't appear to be valid to PhEDEx, " % site
msg += "skipping subscription: %s" % subInfo['id']
logging.error(msg)
self.sendAlert(7, msg = msg)
continue
# Get the phedex node from CMS site
site = self.cmsToPhedexMap[site].get("MSS") or self.cmsToPhedexMap[site]["Disk"]
# Avoid custodial subscriptions to disk nodes
if site not in self.phedexNodes['MSS']: subInfo['custodial'] = 'n'
# Avoid move subscriptions and replica
if subInfo['custodial'] == 'n': subInfo['move'] = 'n'
phedexSub = PhEDExSubscription(subInfo['path'], site,
self.group, priority = subInfo['priority'],
move = subInfo['move'], custodial = subInfo['custodial'],
request_only = subInfo['request_only'], subscriptionId = subInfo['id'])
# Check if the subscription is a duplicate
if phedexSub.matchesExistingSubscription(self.phedex) or \
phedexSub.matchesExistingTransferRequest(self.phedex):
subscriptionsMade.append(subInfo['id'])
continue
# Add it to the list
subs.addSubscription(phedexSub)
# Compact the subscriptions
subs.compact()
for subscription in subs.getSubscriptionList():
try:
xmlData = XMLDrop.makePhEDExXMLForDatasets(self.dbsUrl,
subscription.getDatasetPaths())
logging.debug(str(xmlData))
msg = "Subscribing: %s to %s, with options: " % (subscription.getDatasetPaths(), subscription.getNodes())
msg += "Move: %s, Custodial: %s, Request Only: %s" % (subscription.move, subscription.custodial, subscription.request_only)
logging.info(msg)
self.phedex.subscribe(subscription, xmlData)
except Exception, ex:
logging.error("Something went wrong when communicating with PhEDEx, will try again later.")
logging.error("Exception: %s" % str(ex))
else:
subscriptionsMade.extend(subscription.getSubscriptionIds())
def testSubscriptionList(self):
"""
_SubscriptionList_
Check that we can organize and agreggate correctly a bunch of different subscriptions
in standard scenarios
"""
subList = SubscriptionList()
# Two GEN datasets subscribed to many sites non-custodially and custodially to one
subs = []
genDatasetA = "/DeadlyNeurotoxinOnTestSubjectSim/Run1970-Test-v2/GEN"
subs.append({
"datasetPathList": genDatasetA,
"nodeList": "T1_US_FNAL",
"group": "dataops",
"custodial": "y",
"move": "y"
})
subs.append({
"datasetPathList": genDatasetA,
"nodeList": "T2_IT_Bari",
"group": "dataops",
"request_only": "n"
})
subs.append({
"datasetPathList": genDatasetA,
"nodeList": "T2_CH_CERN",
"group": "dataops",
"request_only": "n"
})
subs.append({
"datasetPathList": genDatasetA,
"nodeList": "T2_US_Wisconsin",
"group": "dataops"
})
genDatasetB = "/NotEnoughEnergyToLieIn1.1V/Run1970-Potato-v2/GEN"
subs.append({
"datasetPathList": genDatasetB,
"nodeList": "T1_IT_CNAF",
"group": "dataops",
"custodial": "y",
"move": "y"
})
subs.append({
"datasetPathList": genDatasetB,
"nodeList": "T2_IT_Bari",
"group": "dataops",
"request_only": "n"
})
subs.append({
"datasetPathList": genDatasetB,
"nodeList": "T2_CH_CERN",
"group": "dataops",
"request_only": "n"
})
subs.append({
"datasetPathList": genDatasetB,
"nodeList": "T2_US_Wisconsin",
"group": "dataops"
})
# RECO,DQM,AOD datasets subscribed custodially to 2 sites
recoDatasetA = '/TestWeightedCubes/Run1970-Test-v2/%s'
recoDatasetB = '/RepulsiveGel/Run1970-Test-v2/%s'
subs.append({
"datasetPathList": recoDatasetA % 'AOD',
"nodeList": "T1_US_FNAL",
"group": "dataops",
"custodial": "y",
"move": "y"
})
subs.append({
"datasetPathList": recoDatasetA % 'DQM',
"nodeList": "T1_US_FNAL",
"group": "dataops",
"custodial": "y",
"move": "y"
})
subs.append({
"datasetPathList": recoDatasetB % 'AOD',
"nodeList": "T1_DE_KIT",
"group": "dataops",
"custodial": "y",
"move": "y"
})
subs.append({
"datasetPathList": recoDatasetB % 'DQM',
"nodeList": "T1_DE_KIT",
"group": "dataops",
"custodial": "y",
"move": "y"
})
for sub in subs:
phedexSub = PhEDExSubscription(**sub)
subList.addSubscription(phedexSub)
# One subscription per node
self.assertEqual(len(subList.getSubscriptionList()), 6)
goldenDatasetLists = [
set([genDatasetA, genDatasetB]),
set([genDatasetA, recoDatasetA % 'AOD', recoDatasetA % 'DQM']),
set([genDatasetB]),
set([recoDatasetB % 'AOD', recoDatasetB % 'DQM'])
]
for sub in subList.getSubscriptionList():
self.assertTrue(set(sub.getDatasetPaths()) in goldenDatasetLists)
subList.compact()
# Compact should have reduced 2 of them to 1
goldenNodeLists = [
set(["T1_US_FNAL"]),
set(["T2_IT_Bari", "T2_CH_CERN"]),
set(["T1_IT_CNAF"]),
set(["T1_DE_KIT"]),
set(["T2_US_Wisconsin"])
]
self.assertEqual(len(subList.getSubscriptionList()), 5)
for sub in subList.getSubscriptionList():
self.assertTrue(set(sub.getNodes()) in goldenNodeLists)
return
def makeTransferRequest(self, wflow):
"""
Send request to PhEDEx and return status of request subscription
This method does the following:
1. return if there is no workflow data to be transferred
2. check if the data input campaign is in the database, skip if not
3. _getValidSites: using the workflow site lists and the campaign configuration,
find a common list of sites (converted to PNNs). If the PNN is out of quota,
it's also removed from this list
4. create the transfer record dictionary
5. for every final node
5.1. if it's a pileup dataset, pick a random node and subscribe the whole dataset
5.2. else, retrieve chunks of blocks to be subscribed (evenly distributed)
5.3. update node usage with the amount of data subscribed
6. re-evaluate nodes with quota exceeded
7. return the transfer record, with a list of transfer IDs
:param wflow: workflow object
:return: boolean whether it succeeded or not, and a subscription dictionary {"dataset":transferIDs}
"""
response = []
success = True
if not (wflow.getParentBlocks() or wflow.getPrimaryBlocks() or wflow.getSecondarySummary()):
self.logger.info("Request %s does not have any further data to transfer", wflow.getName())
return success, response
self.logger.info("Handling data subscriptions for request: %s", wflow.getName())
for dataIn in wflow.getDataCampaignMap():
if dataIn["type"] == "parent":
msg = "Skipping 'parent' data subscription (done with the 'primary' data), for: %s" % dataIn
self.logger.info(msg)
continue
elif dataIn["type"] == "secondary" and dataIn['name'] not in wflow.getSecondarySummary():
# secondary already in place
continue
if dataIn['campaign'] not in self.campaigns:
msg = "Data placement can't proceed because campaign '%s' was not found." % dataIn["campaign"]
msg += " Skipping this workflow until the campaign gets created."
self.logger.warning(msg)
return False, response
nodes = self._getValidSites(wflow, dataIn)
if not nodes:
msg = "There are no RSEs with available space for %s. " % wflow.getName()
msg += "Skipping this workflow until RSEs get enough free space"
self.logger.warning(msg)
return False, response
transRec = newTransferRec(dataIn)
for blocks, dataSize, idx in self._decideDataDestination(wflow, dataIn, len(nodes)):
if not blocks and dataIn["type"] == "primary":
# no valid files in any blocks, it will likely fail in global workqueue
return success, response
if blocks:
subLevel = "block"
data = {dataIn['name']: blocks}
else:
# then it's a dataset level subscription
subLevel = "dataset"
data = None
subscription = PhEDExSubscription(datasetPathList=dataIn['name'],
nodeList=nodes[idx],
group=self.msConfig['quotaAccount'],
level=subLevel,
priority="low",
request_only=self.msConfig["phedexRequestOnly"],
blocks=data,
comments="WMCore MicroService automated subscription")
msg = "Creating '%s' level subscription for %s dataset: %s" % (subscription.level,
dataIn['type'],
dataIn['name'])
if wflow.getParentDataset():
msg += ", where parent blocks have also been added for dataset: %s" % wflow.getParentDataset()
self.logger.info(msg)
if self.msConfig.get('enableDataTransfer', True):
# Force request-only subscription
# to any data transfer going above some threshold (do not auto-approve)
aboveWarningThreshold = self.msConfig.get('warningTransferThreshold') > 0. and \
dataSize > self.msConfig.get('warningTransferThreshold')
if aboveWarningThreshold and subscription.request_only != "y":
subscription.request_only = "y"
# Then make the data subscription, for real!!!
success, transferId = self._subscribeData(subscription, wflow.getName(), dataIn['name'])
if not success:
break
if transferId:
transRec['transferIDs'].add(transferId)
# Warn about data transfer subscriptions going above some treshold
if aboveWarningThreshold:
emailSubject = "[MS] Large pending data transfer under request id: {transferid}".format(
transferid=transferId)
emailMsg = "Workflow: {}\nhas a large amount of ".format(wflow.getName())
emailMsg += "data subscribed: {} TB,\n".format(teraBytes(dataSize))
emailMsg += "for {} data: {}.""".format(dataIn['type'], dataIn['name'])
self.emailAlert.send(emailSubject, emailMsg)
self.logger.info(emailMsg)
# and update some instance caches
self.rseQuotas.updateNodeUsage(nodes[idx], dataSize)
if subLevel == 'dataset':
self.dsetCounter += 1
else:
self.blockCounter += len(blocks)
else:
self.logger.info("DRY-RUN: making subscription: %s", subscription)
transRec['transferIDs'] = list(transRec['transferIDs'])
response.append(transRec)
# once the workflow has been completely processed, update the node usage
self.rseQuotas.evaluateQuotaExceeded()
return success, response
def algorithm(self, parameters):
"""
_algorithm_
Poll the database for datasets and subscribe them.
"""
myThread = threading.currentThread()
myThread.transaction.begin()
# Check for completely unsubscribed datasets
unsubscribedDatasets = self.getUnsubscribed.execute(conn = myThread.transaction.conn,
transaction = True)
if self.safeMode:
partiallySubscribedDatasets = self.getPartiallySubscribed.execute(conn = myThread.transaction.conn,
transaction = True)
unsubscribedDatasets.extend(partiallySubscribedDatasets)
partiallySubscribedSet = set()
for entry in partiallySubscribedDatasets:
partiallySubscribedSet.add(entry["path"])
# Map the datasets to their specs
specDatasetMap = {}
for unsubscribedDataset in unsubscribedDatasets:
datasetPath = unsubscribedDataset["path"]
workflow = unsubscribedDataset["workflow"]
spec = unsubscribedDataset["spec"]
if datasetPath not in specDatasetMap:
specDatasetMap[datasetPath] = []
specDatasetMap[datasetPath].append({"workflow" : workflow, "spec" : spec})
specCache = {}
siteMap = {}
# Distribute the subscriptions by site, type and priority
# This is to make as few subscriptions as possible
# Site map values are dictionaries where the keys are tuples (Prio, Custodial, AutoApprove, Move)
# Where Custodial is boolean, Prio is in ["Low", "Normal", "High"], AutoApprove is boolean and Move is boolean
for dataset in specDatasetMap:
# Aggregate all the different subscription configurations
subInfo = {}
for entry in specDatasetMap[dataset]:
if not entry["spec"]:
# Can't use this spec, there isn't one
continue
# Load spec if not in the cache
if entry["spec"] not in specCache:
helper = WMWorkloadHelper()
try:
helper.load(entry["spec"])
specCache[entry["spec"]] = helper
except Exception:
#Couldn't load it , alert and carry on
msg = "Couldn't load spec: %s" % entry["spec"]
logging.error(msg)
self.sendAlert(7, msg = msg)
continue
#If we are running in safe mode, we need to know if the workflow is ready
# We have the spec, get the info
helper = specCache[entry["spec"]]
workflowSubInfo = helper.getSubscriptionInformation()
datasetSubInfo = workflowSubInfo.get(dataset, None)
if datasetSubInfo and subInfo:
subInfo["CustodialSites"] = extendWithoutDups(subInfo["CustodialSites"], datasetSubInfo["CustodialSites"])
subInfo["NonCustodialSites"] = extendWithoutDups(subInfo["NonCustodialSites"], datasetSubInfo["NonCustodialSites"])
subInfo["AutoApproveSites"] = extendWithoutDups(subInfo["AutoApproveSites"], datasetSubInfo["AutoApproveSites"])
subInfo["Priority"] = solvePrioConflicts(subInfo["Priority"], datasetSubInfo["Priority"])
elif datasetSubInfo:
subInfo = datasetSubInfo
# We now have aggregated subscription information for this dataset in subInfo
# Distribute it by site
if not subInfo:
#Nothing to do, log and continue
msg = "No subscriptions configured for dataset %s" % dataset
logging.warning(msg)
self.markSubscribed.execute(dataset, subscribed = self.terminalSubscriptionState,
conn = myThread.transaction.conn,
transaction = True)
continue
# Make sure that a site is not configured both as non custodial and custodial
# Non-custodial is believed to be the right choice
subInfo["CustodialSites"] = list(set(subInfo["CustodialSites"]) - set(subInfo["NonCustodialSites"]))
for site in subInfo["CustodialSites"]:
if site not in siteMap:
siteMap[site] = {}
if self.safeMode and dataset not in partiallySubscribedSet:
tupleKey = (subInfo["Priority"], True, False, False)
else:
tupleKey = (subInfo["Priority"], True, False, True)
if tupleKey not in siteMap[site]:
siteMap[site][tupleKey] = []
siteMap[site][tupleKey].append(dataset)
# If we are in safe mode and this is a partially subscribed dataset,
# then the non-custodial were done in a previous cycle
if self.safeMode and dataset in partiallySubscribedSet:
self.markSubscribed.execute(dataset, subscribed = self.terminalSubscriptionState,
conn = myThread.transaction.conn,
transaction = True)
continue
for site in subInfo["NonCustodialSites"]:
if site not in siteMap:
siteMap[site] = {}
autoApprove = False
if site in subInfo["AutoApproveSites"]:
autoApprove = True
tupleKey = (subInfo["Priority"], False, autoApprove)
if tupleKey not in siteMap[site]:
siteMap[site][tupleKey] = []
siteMap[site][tupleKey].append(dataset)
self.markSubscribed.execute(dataset, subscribed = 1,
conn = myThread.transaction.conn,
transaction = True)
# Actually request the subscriptions
for site in siteMap:
# Check that the site is valid
if site not in self.cmsToPhedexMap:
msg = "Site %s doesn't appear to be valid to PhEDEx" % site
logging.error(msg)
self.sendAlert(7, msg = msg)
continue
for subscriptionFlavor in siteMap[site]:
datasets = siteMap[site][subscriptionFlavor]
# Check that the site is valid
if "MSS" in self.cmsToPhedexMap[site]:
phedexNode = self.cmsToPhedexMap[site]["MSS"]
else:
phedexNode = self.cmsToPhedexMap[site]["Disk"]
logging.info("Subscribing %s to %s" % (datasets, site))
options = {"custodial" : "n", "requestOnly" : "y",
"priority" : subscriptionFlavor[0].lower(),
"move" : "n"}
if subscriptionFlavor[1]:
options["custodial"] = "y"
if subscriptionFlavor[3]:
options["move"] = "y"
if subscriptionFlavor[2]:
options["requestOnly"] = "n"
newSubscription = PhEDExSubscription(datasets, phedexNode, self.group,
**options)
xmlData = XMLDrop.makePhEDExXMLForDatasets(self.dbsUrl,
newSubscription.getDatasetPaths())
logging.debug(str(xmlData))
self.phedex.subscribe(newSubscription, xmlData)
myThread.transaction.commit()
return
def subscribeDatasets(self):
"""
_subscribeDatasets_
Poll the database for datasets and subscribe them.
"""
logging.info("Starting subscribeDatasets method")
# Check for completely unsubscribed datasets
unsubscribedDatasets = self.getUnsubscribed.execute()
# Keep a list of subscriptions to tick as subscribed in the database
subscriptionsMade = []
# Create a list of subscriptions as defined by the PhEDEx data structures
subs = SubscriptionList()
# Create the subscription objects and add them to the list
# The list takes care of the sorting internally
for subInfo in unsubscribedDatasets:
site = subInfo['site']
if site not in self.phedexNodes['MSS'] and site not in self.phedexNodes['Disk']:
msg = "Site %s doesn't appear to be valid to PhEDEx, " % site
msg += "skipping subscription: %s" % subInfo['id']
logging.error(msg)
self.sendAlert(7, msg = msg)
continue
# Avoid custodial subscriptions to disk nodes
if site not in self.phedexNodes['MSS']:
subInfo['custodial'] = 'n'
# Avoid auto approval in T1 sites
elif site.startswith("T1"):
subInfo['request_only'] = 'y'
phedexSub = PhEDExSubscription(subInfo['path'], site, subInfo['phedex_group'],
priority = subInfo['priority'],
move = subInfo['move'],
custodial = subInfo['custodial'],
request_only = subInfo['request_only'],
subscriptionId = subInfo['id'])
# Check if the subscription is a duplicate
if phedexSub.matchesExistingSubscription(self.phedex) or \
phedexSub.matchesExistingTransferRequest(self.phedex):
subscriptionsMade.append(subInfo['id'])
continue
# Add it to the list
subs.addSubscription(phedexSub)
# Compact the subscriptions
subs.compact()
for subscription in subs.getSubscriptionList():
xmlData = XMLDrop.makePhEDExXMLForDatasets(self.dbsUrl, subscription.getDatasetPaths())
logging.debug("subscribeDatasets XMLData: %s" , xmlData)
logging.info("Subscribing: %s to %s, with options: Move: %s, Custodial: %s, Request Only: %s",
subscription.getDatasetPaths(),
subscription.getNodes(),
subscription.move,
subscription.custodial,
subscription.request_only)
try:
self.phedex.subscribe(subscription, xmlData)
except HTTPException as ex:
logging.error("PhEDEx dataset subscribe failed with HTTPException: %s %s", ex.status, ex.result)
except Exception as ex:
logging.error("PhEDEx dataset subscribe failed with Exception: %s", str(ex))
logging.debug("Traceback: %s", str(traceback.format_exc()))
else:
subscriptionsMade.extend(subscription.getSubscriptionIds())
# Register the result in DBSBuffer
if subscriptionsMade:
self.markSubscribed.execute(subscriptionsMade)
return
def algorithm(self, parameters):
"""
_algorithm_
Poll the database for datasets and subscribe them.
"""
myThread = threading.currentThread()
myThread.transaction.begin()
# Check for completely unsubscribed datasets
unsubscribedDatasets = self.getUnsubscribed.execute(
conn=myThread.transaction.conn, transaction=True)
if self.safeMode:
partiallySubscribedDatasets = self.getPartiallySubscribed.execute(
conn=myThread.transaction.conn, transaction=True)
unsubscribedDatasets.extend(partiallySubscribedDatasets)
partiallySubscribedSet = set()
for entry in partiallySubscribedDatasets:
partiallySubscribedSet.add(entry["path"])
# Map the datasets to their specs
specDatasetMap = {}
for unsubscribedDataset in unsubscribedDatasets:
datasetPath = unsubscribedDataset["path"]
workflow = unsubscribedDataset["workflow"]
spec = unsubscribedDataset["spec"]
if datasetPath not in specDatasetMap:
specDatasetMap[datasetPath] = []
specDatasetMap[datasetPath].append({
"workflow": workflow,
"spec": spec
})
specCache = {}
siteMap = {}
# Distribute the subscriptions by site, type and priority
# This is to make as few subscriptions as possible
# Site map values are dictionaries where the keys are tuples (Prio, Custodial, AutoApprove, Move)
# Where Custodial is boolean, Prio is in ["Low", "Normal", "High"], AutoApprove is boolean and Move is boolean
for dataset in specDatasetMap:
# Aggregate all the different subscription configurations
subInfo = {}
for entry in specDatasetMap[dataset]:
if not entry["spec"]:
# Can't use this spec, there isn't one
continue
# Load spec if not in the cache
if entry["spec"] not in specCache:
helper = WMWorkloadHelper()
try:
helper.load(entry["spec"])
specCache[entry["spec"]] = helper
except Exception:
#Couldn't load it , alert and carry on
msg = "Couldn't load spec: %s" % entry["spec"]
logging.error(msg)
self.sendAlert(7, msg=msg)
continue
#If we are running in safe mode, we need to know if the workflow is ready
# We have the spec, get the info
helper = specCache[entry["spec"]]
workflowSubInfo = helper.getSubscriptionInformation()
datasetSubInfo = workflowSubInfo.get(dataset, None)
if datasetSubInfo and subInfo:
subInfo["CustodialSites"] = extendWithoutDups(
subInfo["CustodialSites"],
datasetSubInfo["CustodialSites"])
subInfo["NonCustodialSites"] = extendWithoutDups(
subInfo["NonCustodialSites"],
datasetSubInfo["NonCustodialSites"])
subInfo["AutoApproveSites"] = extendWithoutDups(
subInfo["AutoApproveSites"],
datasetSubInfo["AutoApproveSites"])
subInfo["Priority"] = solvePrioConflicts(
subInfo["Priority"], datasetSubInfo["Priority"])
elif datasetSubInfo:
subInfo = datasetSubInfo
# We now have aggregated subscription information for this dataset in subInfo
# Distribute it by site
if not subInfo:
#Nothing to do, log and continue
msg = "No subscriptions configured for dataset %s" % dataset
logging.warning(msg)
self.markSubscribed.execute(
dataset,
subscribed=self.terminalSubscriptionState,
conn=myThread.transaction.conn,
transaction=True)
continue
# Make sure that a site is not configured both as non custodial and custodial
# Non-custodial is believed to be the right choice
subInfo["CustodialSites"] = list(
set(subInfo["CustodialSites"]) -
set(subInfo["NonCustodialSites"]))
for site in subInfo["CustodialSites"]:
if site not in siteMap:
siteMap[site] = {}
autoApprove = False
if site in subInfo["AutoApproveSites"]:
autoApprove = True
if self.safeMode and dataset not in partiallySubscribedSet:
tupleKey = (subInfo["Priority"], True, autoApprove, False)
else:
tupleKey = (subInfo["Priority"], True, autoApprove, True)
if tupleKey not in siteMap[site]:
siteMap[site][tupleKey] = []
# Subscriptions are sorted by options, defined by tupleKey
# The tuple key has 3 or 4 entries in this order
# Priority, Custodial, Auto approve, Move (True) or Replica (False)
siteMap[site][tupleKey].append(dataset)
# If we are in safe mode and this is a partially subscribed dataset,
# then the non-custodial were done in a previous cycle
if self.safeMode and dataset in partiallySubscribedSet:
self.markSubscribed.execute(
dataset,
subscribed=self.terminalSubscriptionState,
conn=myThread.transaction.conn,
transaction=True)
continue
for site in subInfo["NonCustodialSites"]:
if site not in siteMap:
siteMap[site] = {}
autoApprove = False
if site in subInfo["AutoApproveSites"]:
autoApprove = True
# Non-custodial is never move, so this tuple has only 3 entries
# TODO: Change tuples to frozensets for clarity
tupleKey = (subInfo["Priority"], False, autoApprove)
if tupleKey not in siteMap[site]:
siteMap[site][tupleKey] = []
siteMap[site][tupleKey].append(dataset)
self.markSubscribed.execute(dataset,
subscribed=1,
conn=myThread.transaction.conn,
transaction=True)
# Actually request the subscriptions
for site in siteMap:
# Check that the site is valid
if site not in self.cmsToPhedexMap:
msg = "Site %s doesn't appear to be valid to PhEDEx" % site
logging.error(msg)
self.sendAlert(7, msg=msg)
continue
for subscriptionFlavor in siteMap[site]:
datasets = siteMap[site][subscriptionFlavor]
# Check that the site is valid
isMSS = False
if "MSS" in self.cmsToPhedexMap[site]:
isMSS = True
phedexNode = self.cmsToPhedexMap[site]["MSS"]
else:
phedexNode = self.cmsToPhedexMap[site]["Disk"]
logging.info("Subscribing %s to %s" % (datasets, site))
options = {
"custodial": "n",
"requestOnly": "y",
"priority": subscriptionFlavor[0].lower(),
"move": "n"
}
if subscriptionFlavor[1] and isMSS:
# Custodial subscriptions are only allowed in MSS nodes
# If custodial is requested on Non-MSS it fallsback to a non-custodial subscription
options["custodial"] = "y"
if subscriptionFlavor[3] and not self.replicaOnly:
options["move"] = "y"
if subscriptionFlavor[2]:
options["requestOnly"] = "n"
logging.info(
"Request options: Custodial - %s, Move - %s, Request Only - %s"
% (options["custodial"].upper(), options["move"].upper(),
options["requestOnly"].upper()))
newSubscription = PhEDExSubscription(datasets, phedexNode,
self.group, **options)
xmlData = XMLDrop.makePhEDExXMLForDatasets(
self.dbsUrl, newSubscription.getDatasetPaths())
logging.debug(str(xmlData))
self.phedex.subscribe(newSubscription, xmlData)
myThread.transaction.commit()
return
