def deleteBlocksPhEDExCalls(self, location, blocksToDelete): """ _deleteBlocksPhEDExCalls_ actual PhEDEx calls for block deletion """ deletion = PhEDExDeletion(blocksToDelete.keys(), location, level = 'block', comments = "WMAgent blocks auto-delete from %s" % location, blocks = blocksToDelete) xmlData = XMLDrop.makePhEDExXMLForBlocks(self.dbsUrl, deletion.getDatasetsAndBlocks()) logging.debug("deleteBlocks XMLData: %s", xmlData) try: response = self.phedex.delete(deletion, xmlData) requestId = response['phedex']['request_created'][0]['id'] # auto-approve deletion request self.phedex.updateRequest(requestId, 'approve', location) except HTTPException as ex: logging.error("PhEDEx block delete/approval failed with HTTPException: %s %s", ex.status, ex.result) except Exception as ex: logging.error("PhEDEx block delete/approval failed with Exception: %s", str(ex)) logging.debug("Traceback: %s", str(traceback.format_exc())) else: binds = [] for dataset in blocksToDelete: for blockName in blocksToDelete[dataset]: binds.append( { 'DELETED': 1, 'BLOCKNAME': blockName } ) self.markBlocksDeleted.execute(binds) return
def subscribe(self, subscription): """ _subscribe_ xmlData = XMLDrop.makePhEDExXMLForDatasets(dbsUrl, subscription.getDatasetPaths()) Subscription is PhEDEX subscription structure """ callname = 'subscribe' args = {} args['node'] = [] for node in subscription.nodes: args['node'].append(node) args['comments'] = subscription.comments args['level'] = subscription.level args['priority'] = subscription.priority args['move'] = subscription.move args['static'] = subscription.static args['custodial'] = subscription.custodial args['group'] = subscription.group args['request_only'] = subscription.request_only if args['level'] == 'dataset': xmlData = XMLDrop.makePhEDExXMLForDatasets( self.dbsUrl, list(subscription.datasetPaths)) else: # block xmlData = XMLDrop.makePhEDExXMLForBlocks( self.dbsUrl, subscription.getDatasetsAndBlocks()) args['data'] = xmlData return self._getResult(callname, args=args, verb="POST")
def deleteBlocksPhEDExCalls(self, location, blocksToDelete): """ _deleteBlocksPhEDExCalls_ actual PhEDEx calls for block deletion """ deletion = PhEDExDeletion(blocksToDelete.keys(), location, level='block', comments="WMAgent blocks auto-delete from %s" % location, blocks=blocksToDelete) xmlData = XMLDrop.makePhEDExXMLForBlocks(self.dbsUrl, deletion.getDatasetsAndBlocks()) logging.debug("deleteBlocks XMLData: %s", xmlData) try: response = self.phedex.delete(deletion, xmlData) requestId = response['phedex']['request_created'][0]['id'] # auto-approve deletion request self.phedex.updateRequest(requestId, 'approve', location) except HTTPException as ex: logging.error("PhEDEx block delete/approval failed with HTTPException: %s %s", ex.status, ex.result) except Exception as ex: logging.error("PhEDEx block delete/approval failed with Exception: %s", str(ex)) logging.debug("Traceback: %s", str(traceback.format_exc())) else: binds = [] for dataset in blocksToDelete: for blockName in blocksToDelete[dataset]: binds.append({'DELETED': 1, 'BLOCKNAME': blockName}) self.markBlocksDeleted.execute(binds) return
def phedex(): phedexIn = PhEDEx(dict = {'endpoint' : 'https://cmsweb.cern.ch/phedex/datasvc/json/dev/', 'logger' : logging}, responseType = "json") # requests = phedex.getRequestList(dataset = ['/TauParked/Run2012C-LogError-22Jan2013-v1/RAW-RECO'], # node = 'T2_RU_ITEP')['phedex']['request'] # for request in requests: # requestId = request['id'] # request = phedex.getTransferRequests(request = requestId)['phedex']['request'] # if request: # request = request[0] # print request # x = PhEDExSubscription('/TauParked/Run2012C-22Jan2013-v1/AOD', # 'T1_US_FNAL_MSS', 'DataOps', 'dataset', 'low', 'n', 'n', 'n', 'y', subscriptionId = 1) # print x.matchesExistingTransferRequest(phedex) # print x.matchesExistingSubscription(phedex) deletion = PhEDExDeletion('/071103be-7d80-11e0-90de-00163e010039/PromptReco-v1/RECO', 'T1_CH_CERN_Buffer', level = 'block', comments = 'Blocks automatically deleted from T2_CH_CERN as it has already been processed and transferred to a custodial location', blocks = {'/071103be-7d80-11e0-90de-00163e010039/PromptReco-v1/RECO' : ['/071103be-7d80-11e0-90de-00163e010039/PromptReco-v1/RECO#075ea9e8-7d80-11e0-90de-00163e010039']}) xmlData = XMLDrop.makePhEDExXMLForBlocks('http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet', deletion.getDatasetsAndBlocks()) print str(xmlData) response = phedexIn.delete(deletion, xmlData) print response requestId = response['phedex']['request_created'][0]['id'] phedexIn.updateRequest(requestId, 'approve', 'T1_CH_CERN_Buffer')
def deleteBlocksPhEDExCalls(self, location, blocksToDelete): """ _deleteBlocksPhEDExCalls_ actual PhEDEx calls for block deletion """ deletion = PhEDExDeletion(blocksToDelete.keys(), location, level = 'block', comments = "WMAgent blocks auto-delete from %s" % location, blocks = blocksToDelete) try: xmlData = XMLDrop.makePhEDExXMLForBlocks(self.dbsUrl, deletion.getDatasetsAndBlocks()) logging.debug(str(xmlData)) response = self.phedex.delete(deletion, xmlData) requestId = response['phedex']['request_created'][0]['id'] # auto-approve deletion request self.phedex.updateRequest(requestId, 'approve', location) binds = [] for dataset in blocksToDelete: for blockName in blocksToDelete[dataset]: binds.append( { 'DELETED' : 1, 'BLOCKNAME' : blockName } ) self.markBlocksDeleted.execute(binds, transaction = False) except Exception as ex: logging.error("Something went wrong when communicating with PhEDEx, will try again later.") logging.error("Exception: %s", str(ex)) return
def subscribeTier0Blocks(self): """ _subscribeTier0Blocks_ Subscribe blocks to the Tier-0 where a replica subscription already exists. All Tier-0 subscriptions are move, custodial and autoapproved with high priority. """ myThread = threading.currentThread() myThread.transaction.begin() # Check for candidate blocks for subscription blocksToSubscribe = self.getUnsubscribedBlocks.execute(node = 'T0_CH_CERN', conn = myThread.transaction.conn, transaction = True) if not blocksToSubscribe: return # For the blocks we don't really care about the subscription options # We are subscribing all blocks with the same recipe. subscriptionMap = {} for subInfo in blocksToSubscribe: dataset = subInfo['path'] if dataset not in subscriptionMap: subscriptionMap[dataset] = [] subscriptionMap[dataset].append(subInfo['blockname']) site = 'T0_CH_CERN' custodial = 'y' request_only = 'n' move = 'y' priority = 'High' # Get the phedex node phedexNode = self.cmsToPhedexMap[site]["MSS"] logging.error("Subscribing %d blocks, from %d datasets to the Tier-0" % (len(subscriptionMap), sum([len(x) for x in subscriptionMap.values()]))) newSubscription = PhEDExSubscription(subscriptionMap.keys(), phedexNode, self.group, custodial = custodial, request_only = request_only, move = move, priority = priority, level = 'block', blocks = subscriptionMap) # TODO: Check for blocks already subscribed try: xmlData = XMLDrop.makePhEDExXMLForBlocks(self.dbsUrl, newSubscription.getDatasetsAndBlocks()) logging.debug(str(xmlData)) self.phedex.subscribe(newSubscription, xmlData) except Exception, ex: logging.error("Something went wrong when communicating with PhEDEx, will try again later.") logging.error("Exception: %s" % str(ex))
def delete(self, deletion): """ _delete_ Deletion is a PhEDEX deletion structure """ callname = 'delete' args = {} args['node'] = [] for node in deletion.nodes: args['node'].append(node) xmlData = XMLDrop.makePhEDExXMLForBlocks(self.dbsUrl, deletion.getDatasetsAndBlocks()) args['data'] = xmlData args['level'] = deletion.level args['rm_subscriptions'] = deletion.subscriptions args['comments'] = deletion.comments return self._getResult(callname, args=args, verb="POST")
def deleteBlocksPhEDExCalls(self, location, blocksToDelete): """ _deleteBlocksPhEDExCalls_ actual PhEDEx calls for block deletion """ deletion = PhEDExDeletion( blocksToDelete.keys(), location, level='block', comments="WMAgent blocks auto-delete from %s" % location, blocks=blocksToDelete) try: xmlData = XMLDrop.makePhEDExXMLForBlocks( self.dbsUrl, deletion.getDatasetsAndBlocks()) logging.debug(str(xmlData)) response = self.phedex.delete(deletion, xmlData) requestId = response['phedex']['request_created'][0]['id'] # auto-approve deletion request self.phedex.updateRequest(requestId, 'approve', location) binds = [] for dataset in blocksToDelete: for blockName in blocksToDelete[dataset]: binds.append({'DELETED': 1, 'BLOCKNAME': blockName}) self.markBlocksDeleted.execute(binds, transaction=False) except Exception as ex: logging.error( "Something went wrong when communicating with PhEDEx, will try again later." ) logging.error("Exception: %s", str(ex)) return