def __getRequestString(self, requestName): """ Obtain the string for request (including all sub-requests) :param self: self reference :param str requestName: request name """ self.log.info("__getRequestString: Attempting to get string for %s." % requestName) res = self.__locateRequest(requestName) if not res["OK"]: return res subRequestPaths = res["Value"] try: oRequest = RequestContainer(init=False) for subRequestPath in subRequestPaths: res = self.__readSubRequestString(subRequestPath) if not res["OK"]: return res subRequestString = res["Value"] tempRequest = RequestContainer(subRequestString) oRequest.setRequestAttributes(tempRequest.getRequestAttributes()["Value"]) oRequest.update(tempRequest) requestString = oRequest.toXML()["Value"] self.log.info("__getRequestString: Successfully obtained string for %s." % requestName) result = S_OK(requestString) result["Request"] = oRequest return result except Exception, error: errStr = "__getRequestString: Exception while obtaining request string." self.log.exception(errStr, requestName, lException=error) return S_ERROR(errStr)
def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ): """ Set removal request with the given credentials """ request = RequestContainer() request.setRequestAttributes( { 'OwnerDN':ownerDN, 'OwnerGroup':ownerGroup } ) requestName = os.path.basename( lfn ).strip()+'_removal_request.xml' request.setRequestName( requestName ) request.setSourceComponent( 'JobCleaningAgent' ) removalDict = {'Attributes':{ 'Operation':'removeFile', 'TargetSE':'', 'ExecutionOrder':0 } } result = request.addSubRequest( removalDict, 'removal' ) if not result['OK']: return result index = result['Value'] fileDict = { 'LFN':lfn, 'PFN':'', 'Status':'Waiting' } request.setSubRequestFiles( index, 'removal', [fileDict] ) client = RequestClient() result = request.toXML() if not result['OK']: return result xmlRequest = result['Value'] result = client.setRequest( requestName, xmlRequest ) return result
def __getRequestString(self, requestName): """ Obtain the string for request (including all sub-requests) """ gLogger.info( "RequestDBFile.__getRequestString: Attempting to get string for %s." % requestName) res = self.__locateRequest(requestName) if not res['OK']: return res subRequestPaths = res['Value'] try: oRequest = RequestContainer(init=False) for subRequestPath in subRequestPaths: res = self.__readSubRequestString(subRequestPath) if not res['OK']: return res subRequestString = res['Value'] tempRequest = RequestContainer(subRequestString) #,init=False) oRequest.setRequestAttributes( tempRequest.getRequestAttributes()['Value']) oRequest.update(tempRequest) requestString = oRequest.toXML()['Value'] gLogger.info( "RequestDBFile.__getRequestString: Successfully obtained string for %s." % requestName) result = S_OK(requestString) result['Request'] = oRequest return result except Exception, x: errStr = "RequestDBFile.__getRequestString: Exception while obtaining request string." gLogger.exception(errStr, requestName, lException=x) return S_ERROR(errStr)
def __getRequestString( self, requestName ): """ Obtain the string for request (including all sub-requests) """ gLogger.info( "RequestDBFile.__getRequestString: Attempting to get string for %s." % requestName ) res = self.__locateRequest( requestName ) if not res['OK']: return res subRequestPaths = res['Value'] try: oRequest = RequestContainer( init = False ) for subRequestPath in subRequestPaths: res = self.__readSubRequestString( subRequestPath ) if not res['OK']: return res subRequestString = res['Value'] tempRequest = RequestContainer( subRequestString )#,init=False) oRequest.setRequestAttributes( tempRequest.getRequestAttributes()['Value'] ) oRequest.update( tempRequest ) requestString = oRequest.toXML()['Value'] gLogger.info( "RequestDBFile.__getRequestString: Successfully obtained string for %s." % requestName ) result = S_OK( requestString ) result['Request'] = oRequest return result except Exception, x: errStr = "RequestDBFile.__getRequestString: Exception while obtaining request string." gLogger.exception( errStr, requestName, lException = x ) return S_ERROR( errStr )
def prepareTransformationTasks(self, transBody, taskDict, owner='', ownerGroup=''): requestType = 'transfer' requestOperation = 'replicateAndRegister' try: requestType, requestOperation = transBody.split(';') except: pass for taskID in sortList(taskDict.keys()): paramDict = taskDict[taskID] transID = paramDict['TransformationID'] oRequest = RequestContainer(init=False) subRequestIndex = oRequest.initiateSubRequest(requestType)['Value'] attributeDict = { 'Operation': requestOperation, 'TargetSE': paramDict['TargetSE'] } oRequest.setSubRequestAttributes(subRequestIndex, requestType, attributeDict) files = [] for lfn in paramDict['InputData'].split(';'): files.append({'LFN': lfn}) oRequest.setSubRequestFiles(subRequestIndex, requestType, files) requestName = str(transID).zfill(8) + '_' + str(taskID).zfill(8) oRequest.setRequestAttributes({'RequestName': requestName}) taskDict[taskID]['TaskObject'] = oRequest.toXML()['Value'] return S_OK(taskDict)
def __setRemovalRequest(self, lfn, ownerDN, ownerGroup): """ Set removal request with the given credentials """ request = RequestContainer() request.setRequestAttributes({ 'OwnerDN': ownerDN, 'OwnerGroup': ownerGroup }) requestName = os.path.basename(lfn).strip() + '_removal_request.xml' request.setRequestName(requestName) request.setSourceComponent('JobCleaningAgent') removalDict = { 'Attributes': { 'Operation': 'removeFile', 'TargetSE': '', 'ExecutionOrder': 0 } } result = request.addSubRequest(removalDict, 'removal') if not result['OK']: return result index = result['Value'] fileDict = {'LFN': lfn, 'PFN': '', 'Status': 'Waiting'} request.setSubRequestFiles(index, 'removal', [fileDict]) client = RequestClient() result = request.toXML() if not result['OK']: return result xmlRequest = result['Value'] result = client.setRequest(requestName, xmlRequest) return result
def __getRequestString(self, requestName): """ Obtain the string for request (including all sub-requests) :param self: self reference :param str requestName: request name """ self.log.info("__getRequestString: Attempting to get string for %s." % requestName) res = self.__locateRequest(requestName) if not res['OK']: return res subRequestPaths = res['Value'] try: oRequest = RequestContainer(init=False) for subRequestPath in subRequestPaths: res = self.__readSubRequestString(subRequestPath) if not res['OK']: return res subRequestString = res['Value'] tempRequest = RequestContainer(subRequestString) oRequest.setRequestAttributes( tempRequest.getRequestAttributes()['Value']) oRequest.update(tempRequest) requestString = oRequest.toXML()['Value'] self.log.info( "__getRequestString: Successfully obtained string for %s." % requestName) result = S_OK(requestString) result['Request'] = oRequest return result except Exception, error: errStr = "__getRequestString: Exception while obtaining request string." self.log.exception(errStr, requestName, lException=error) return S_ERROR(errStr)
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '' ): requestType = 'transfer' requestOperation = 'replicateAndRegister' try: requestType, requestOperation = transBody.split( ';' ) except: pass for taskID in sortList( taskDict.keys() ): paramDict = taskDict[taskID] transID = paramDict['TransformationID'] oRequest = RequestContainer( init = False ) subRequestIndex = oRequest.initiateSubRequest( requestType )['Value'] attributeDict = {'Operation':requestOperation, 'TargetSE':paramDict['TargetSE']} oRequest.setSubRequestAttributes( subRequestIndex, requestType, attributeDict ) files = [] for lfn in paramDict['InputData'].split( ';' ): files.append( {'LFN':lfn} ) oRequest.setSubRequestFiles( subRequestIndex, requestType, files ) requestName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 ) oRequest.setRequestAttributes( {'RequestName':requestName} ) taskDict[taskID]['TaskObject'] = oRequest.toXML()['Value'] return S_OK( taskDict )
requestOperation = 'removeFile' for lfnList in breakListIntoChunks(lfns, 100): oRequest = RequestContainer() subRequestIndex = oRequest.initiateSubRequest(requestType)['Value'] attributeDict = {'Operation': requestOperation, 'TargetSE': targetSE} oRequest.setSubRequestAttributes(subRequestIndex, requestType, attributeDict) files = [] for lfn in lfnList: files.append({'LFN': lfn}) oRequest.setSubRequestFiles(subRequestIndex, requestType, files) requestName = "%s_%s" % (md5(repr(time.time())).hexdigest()[:16], md5(repr(time.time())).hexdigest()[:16]) oRequest.setRequestAttributes({'RequestName': requestName}) DIRAC.gLogger.info(oRequest.toXML()['Value']) result = requestClient.setRequest(requestName, oRequest.toXML()['Value']) if result['OK']: print 'Submitted Request:', result['Value'] else: print 'Failed to submit Request', result['Message'] if monitor: requestID = result['Value'] while True: result = requestClient.getRequestStatus(requestID) if not result['OK']: Script.gLogger.error(result['Message']) break
def readRequestsForJobs( self, jobIDs ): """ read and return Requests for jobs :param mixed jobIDs: list with jobIDs or long JobIDs """ if type(jobIDs) != list: return S_ERROR("RequestDB: wrong format for jobIDs argument, got %s, expecting a list" ) # make sure list is uniqe and has only longs jobIDs = list( set( [ int(jobID) for jobID in jobIDs if int(jobID) != 0 ] ) ) reqCols = [ "RequestID", "RequestName", "JobID", "Status", "OwnerDN", "OwnerGroup", "DIRACSetup", "SourceComponent", "CreationTime", "SubmissionTime", "LastUpdate" ] subCols = [ "SubRequestID", "Operation", "Arguments", "RequestType", "ExecutionOrder", "Error", "SourceSE", "TargetSE", "Catalogue", "CreationTime", "SubmissionTime", "LastUpdate" ] fileCols = [ "FileID", "LFN", "Size", "PFN", "GUID", "Md5", "Addler", "Attempt", "Status" , "Error" ] requestNames = self.getRequestForJobs( jobIDs ) if not requestNames["OK"]: return requestNames requestNames = requestNames["Value"] ## this will be returned retDict = { "Successful" : dict(), "Failed" : dict() } for jobID in jobIDs: ## missing requests if jobID not in requestNames: retDict["Failed"][jobID] = "Request not found" continue requestName = requestNames[jobID] ## get request queryStr = "SELECT %s FROM Requests WHERE RequestName = '%s';" % ( ",".join( reqCols ), requestName ) queryRes = self._query( queryStr ) if not queryRes["OK"]: retDict["Failed"][jobID] = queryRes["Message"] continue queryRes = queryRes["Value"] if queryRes["Value"] else None if not queryRes: retDict["Failed"][jobID] = "Unable to read request attributes." continue requestObj = RequestContainer( init=False ) reqAttrs = dict( zip( reqCols, queryRes[0] ) ) requestObj.setRequestAttributes( reqAttrs ) queryStr = "SELECT %s FROM `SubRequests` WHERE `RequestID`=%s;" % ( ",".join(subCols), reqAttrs["RequestID"] ) queryRes = self._query( queryStr ) if not queryRes["OK"]: retDict["Failed"][jobID] = queryRes["Message"] continue queryRes = queryRes["Value"] if queryRes["Value"] else None if not queryRes: retDict["Failed"][jobID] = "Unable to read subrequest attributes." continue ## get sub-requests for recTuple in queryRes: subReqAttrs = dict( zip( subCols, recTuple ) ) subType = subReqAttrs["RequestType"] subReqAttrs["ExecutionOrder"] = int( subReqAttrs["ExecutionOrder"] ) del subReqAttrs["RequestType"] index = requestObj.initiateSubRequest( subType ) index = index["Value"] requestObj.setSubRequestAttributes( index, subType, subReqAttrs ) ## get files subFiles = [] fileQuery = "SELECT %s FROM `Files` WHERE `SubRequestID` = %s ORDER BY `FileID`;" % ( ",".join(fileCols), subReqAttrs["SubRequestID"] ) fileQueryRes = self._query( fileQuery ) if fileQueryRes["OK"] and fileQueryRes["Value"]: for fileRec in fileQueryRes["Value"]: subFiles.append( dict( zip(fileCols, fileRec) ) ) if subFiles: requestObj.setSubRequestFiles( index, subType, subFiles ) retDict["Successful"][jobID] = requestObj.toXML()["Value"] return S_OK( retDict )
reqClient = ReqClient() requestType = 'transfer' requestOperation = 'replicateAndRegister' for lfnList in breakListIntoChunks( lfns, 100 ): oRequest = RequestContainer() subRequestIndex = oRequest.initiateSubRequest( requestType )['Value'] attributeDict = {'Operation':requestOperation, 'TargetSE':targetSE} oRequest.setSubRequestAttributes( subRequestIndex, requestType, attributeDict ) files = [] for lfn in lfnList: files.append( {'LFN':lfn} ) oRequest.setSubRequestFiles( subRequestIndex, requestType, files ) requestName = "%s_%s" % ( md5( repr( time.time() ) ).hexdigest()[:16], md5( repr( time.time() ) ).hexdigest()[:16] ) oRequest.setRequestAttributes( {'RequestName':requestName} ) DIRAC.gLogger.info( oRequest.toXML()['Value'] ) result = reqClient.setRequest( requestName, oRequest.toXML()['Value'] ) if result['OK']: print 'Submitted Request:', result['Value'] else: print 'Failed to submit Request', result['Message'] if monitor: requestID = result['Value'] while True: result = reqClient.getRequestStatus( requestID ) if not result['OK']: Script.gLogger.error( result['Message'] ) break
print Script.showHelp() from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer from DIRAC.RequestManagementSystem.Client.RequestClient import RequestClient requestClient = RequestClient() requestType = "transfer" requestOperation = "replicateAndRegister" for lfnList in breakListIntoChunks(lfns, 100): oRequest = RequestContainer() subRequestIndex = oRequest.initiateSubRequest(requestType)["Value"] attributeDict = {"Operation": requestOperation, "TargetSE": targetSE} oRequest.setSubRequestAttributes(subRequestIndex, requestType, attributeDict) files = [] for lfn in lfnList: files.append({"LFN": lfn}) oRequest.setSubRequestFiles(subRequestIndex, requestType, files) requestName = "%s_%s" % (md5(repr(time.time())).hexdigest()[:16], md5(repr(time.time())).hexdigest()[:16]) oRequest.setRequestAttributes({"RequestName": requestName}) DIRAC.gLogger.info(oRequest.toXML()["Value"]) result = requestClient.setRequest(requestName, oRequest.toXML()["Value"]) if result["OK"]: print "Submitted Request:", result["Value"] else: print "Failed to submit Request", result["Message"]