def deleteBlocksPhEDExCalls(self, location, blocksToDelete):
"""
_deleteBlocksPhEDExCalls_
actual PhEDEx calls for block deletion
"""
deletion = PhEDExDeletion(blocksToDelete.keys(), location,
level = 'block',
comments = "WMAgent blocks auto-delete from %s" % location,
blocks = blocksToDelete)
xmlData = XMLDrop.makePhEDExXMLForBlocks(self.dbsUrl,
deletion.getDatasetsAndBlocks())
logging.debug("deleteBlocks XMLData: %s", xmlData)
try:
response = self.phedex.delete(deletion, xmlData)
requestId = response['phedex']['request_created'][0]['id']
# auto-approve deletion request
self.phedex.updateRequest(requestId, 'approve', location)
except HTTPException as ex:
logging.error("PhEDEx block delete/approval failed with HTTPException: %s %s", ex.status, ex.result)
except Exception as ex:
logging.error("PhEDEx block delete/approval failed with Exception: %s", str(ex))
logging.debug("Traceback: %s", str(traceback.format_exc()))
else:
binds = []
for dataset in blocksToDelete:
for blockName in blocksToDelete[dataset]:
binds.append( { 'DELETED': 1,
'BLOCKNAME': blockName } )
self.markBlocksDeleted.execute(binds)
return
def subscribe(self, subscription):
"""
_subscribe_
xmlData = XMLDrop.makePhEDExXMLForDatasets(dbsUrl, subscription.getDatasetPaths())
Subscription is PhEDEX subscription structure
"""
callname = 'subscribe'
args = {}
args['node'] = []
for node in subscription.nodes:
args['node'].append(node)
args['comments'] = subscription.comments
args['level'] = subscription.level
args['priority'] = subscription.priority
args['move'] = subscription.move
args['static'] = subscription.static
args['custodial'] = subscription.custodial
args['group'] = subscription.group
args['request_only'] = subscription.request_only
if args['level'] == 'dataset':
xmlData = XMLDrop.makePhEDExXMLForDatasets(
self.dbsUrl, list(subscription.datasetPaths))
else: # block
xmlData = XMLDrop.makePhEDExXMLForBlocks(
self.dbsUrl, subscription.getDatasetsAndBlocks())
args['data'] = xmlData
return self._getResult(callname, args=args, verb="POST")
def deleteBlocksPhEDExCalls(self, location, blocksToDelete):
"""
_deleteBlocksPhEDExCalls_
actual PhEDEx calls for block deletion
"""
deletion = PhEDExDeletion(blocksToDelete.keys(), location,
level='block',
comments="WMAgent blocks auto-delete from %s" % location,
blocks=blocksToDelete)
xmlData = XMLDrop.makePhEDExXMLForBlocks(self.dbsUrl,
deletion.getDatasetsAndBlocks())
logging.debug("deleteBlocks XMLData: %s", xmlData)
try:
response = self.phedex.delete(deletion, xmlData)
requestId = response['phedex']['request_created'][0]['id']
# auto-approve deletion request
self.phedex.updateRequest(requestId, 'approve', location)
except HTTPException as ex:
logging.error("PhEDEx block delete/approval failed with HTTPException: %s %s", ex.status, ex.result)
except Exception as ex:
logging.error("PhEDEx block delete/approval failed with Exception: %s", str(ex))
logging.debug("Traceback: %s", str(traceback.format_exc()))
else:
binds = []
for dataset in blocksToDelete:
for blockName in blocksToDelete[dataset]:
binds.append({'DELETED': 1,
'BLOCKNAME': blockName})
self.markBlocksDeleted.execute(binds)
return
def phedex():
phedexIn = PhEDEx(dict = {'endpoint' : 'https://cmsweb.cern.ch/phedex/datasvc/json/dev/',
'logger' : logging},
responseType = "json")
# requests = phedex.getRequestList(dataset = ['/TauParked/Run2012C-LogError-22Jan2013-v1/RAW-RECO'],
# node = 'T2_RU_ITEP')['phedex']['request']
# for request in requests:
# requestId = request['id']
# request = phedex.getTransferRequests(request = requestId)['phedex']['request']
# if request:
# request = request[0]
# print request
# x = PhEDExSubscription('/TauParked/Run2012C-22Jan2013-v1/AOD',
# 'T1_US_FNAL_MSS', 'DataOps', 'dataset', 'low', 'n', 'n', 'n', 'y', subscriptionId = 1)
# print x.matchesExistingTransferRequest(phedex)
# print x.matchesExistingSubscription(phedex)
deletion = PhEDExDeletion('/071103be-7d80-11e0-90de-00163e010039/PromptReco-v1/RECO',
'T1_CH_CERN_Buffer',
level = 'block',
comments = 'Blocks automatically deleted from T2_CH_CERN as it has already been processed and transferred to a custodial location',
blocks = {'/071103be-7d80-11e0-90de-00163e010039/PromptReco-v1/RECO' :
['/071103be-7d80-11e0-90de-00163e010039/PromptReco-v1/RECO#075ea9e8-7d80-11e0-90de-00163e010039']})
xmlData = XMLDrop.makePhEDExXMLForBlocks('http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet',
deletion.getDatasetsAndBlocks())
print str(xmlData)
response = phedexIn.delete(deletion, xmlData)
print response
requestId = response['phedex']['request_created'][0]['id']
phedexIn.updateRequest(requestId, 'approve', 'T1_CH_CERN_Buffer')
def deleteBlocksPhEDExCalls(self, location, blocksToDelete):
"""
_deleteBlocksPhEDExCalls_
actual PhEDEx calls for block deletion
"""
deletion = PhEDExDeletion(blocksToDelete.keys(), location,
level = 'block',
comments = "WMAgent blocks auto-delete from %s" % location,
blocks = blocksToDelete)
try:
xmlData = XMLDrop.makePhEDExXMLForBlocks(self.dbsUrl,
deletion.getDatasetsAndBlocks())
logging.debug(str(xmlData))
response = self.phedex.delete(deletion, xmlData)
requestId = response['phedex']['request_created'][0]['id']
# auto-approve deletion request
self.phedex.updateRequest(requestId, 'approve', location)
binds = []
for dataset in blocksToDelete:
for blockName in blocksToDelete[dataset]:
binds.append( { 'DELETED' : 1,
'BLOCKNAME' : blockName } )
self.markBlocksDeleted.execute(binds, transaction = False)
except Exception as ex:
logging.error("Something went wrong when communicating with PhEDEx, will try again later.")
logging.error("Exception: %s", str(ex))
return
def subscribeTier0Blocks(self):
"""
_subscribeTier0Blocks_
Subscribe blocks to the Tier-0 where a replica subscription
already exists. All Tier-0 subscriptions are move, custodial
and autoapproved with high priority.
"""
myThread = threading.currentThread()
myThread.transaction.begin()
# Check for candidate blocks for subscription
blocksToSubscribe = self.getUnsubscribedBlocks.execute(node = 'T0_CH_CERN',
conn = myThread.transaction.conn,
transaction = True)
if not blocksToSubscribe:
return
# For the blocks we don't really care about the subscription options
# We are subscribing all blocks with the same recipe.
subscriptionMap = {}
for subInfo in blocksToSubscribe:
dataset = subInfo['path']
if dataset not in subscriptionMap:
subscriptionMap[dataset] = []
subscriptionMap[dataset].append(subInfo['blockname'])
site = 'T0_CH_CERN'
custodial = 'y'
request_only = 'n'
move = 'y'
priority = 'High'
# Get the phedex node
phedexNode = self.cmsToPhedexMap[site]["MSS"]
logging.error("Subscribing %d blocks, from %d datasets to the Tier-0" % (len(subscriptionMap), sum([len(x) for x in subscriptionMap.values()])))
newSubscription = PhEDExSubscription(subscriptionMap.keys(),
phedexNode, self.group,
custodial = custodial,
request_only = request_only,
move = move,
priority = priority,
level = 'block',
blocks = subscriptionMap)
# TODO: Check for blocks already subscribed
try:
xmlData = XMLDrop.makePhEDExXMLForBlocks(self.dbsUrl,
newSubscription.getDatasetsAndBlocks())
logging.debug(str(xmlData))
self.phedex.subscribe(newSubscription, xmlData)
except Exception, ex:
logging.error("Something went wrong when communicating with PhEDEx, will try again later.")
logging.error("Exception: %s" % str(ex))
def delete(self, deletion):
"""
_delete_
Deletion is a PhEDEX deletion structure
"""
callname = 'delete'
args = {}
args['node'] = []
for node in deletion.nodes:
args['node'].append(node)
xmlData = XMLDrop.makePhEDExXMLForBlocks(self.dbsUrl, deletion.getDatasetsAndBlocks())
args['data'] = xmlData
args['level'] = deletion.level
args['rm_subscriptions'] = deletion.subscriptions
args['comments'] = deletion.comments
return self._getResult(callname, args=args, verb="POST")
def delete(self, deletion):
"""
_delete_
Deletion is a PhEDEX deletion structure
"""
callname = 'delete'
args = {}
args['node'] = []
for node in deletion.nodes:
args['node'].append(node)
xmlData = XMLDrop.makePhEDExXMLForBlocks(self.dbsUrl, deletion.getDatasetsAndBlocks())
args['data'] = xmlData
args['level'] = deletion.level
args['rm_subscriptions'] = deletion.subscriptions
args['comments'] = deletion.comments
return self._getResult(callname, args=args, verb="POST")
def deleteBlocksPhEDExCalls(self, location, blocksToDelete):
"""
_deleteBlocksPhEDExCalls_
actual PhEDEx calls for block deletion
"""
deletion = PhEDExDeletion(
blocksToDelete.keys(),
location,
level='block',
comments="WMAgent blocks auto-delete from %s" % location,
blocks=blocksToDelete)
try:
xmlData = XMLDrop.makePhEDExXMLForBlocks(
self.dbsUrl, deletion.getDatasetsAndBlocks())
logging.debug(str(xmlData))
response = self.phedex.delete(deletion, xmlData)
requestId = response['phedex']['request_created'][0]['id']
# auto-approve deletion request
self.phedex.updateRequest(requestId, 'approve', location)
binds = []
for dataset in blocksToDelete:
for blockName in blocksToDelete[dataset]:
binds.append({'DELETED': 1, 'BLOCKNAME': blockName})
self.markBlocksDeleted.execute(binds, transaction=False)
except Exception as ex:
logging.error(
"Something went wrong when communicating with PhEDEx, will try again later."
)
logging.error("Exception: %s", str(ex))
return
