def generateFailoverFile(self): """ Retrieve the accumulated reporting request, and produce a JSON file that is consumed by the JobWrapper """ reportRequest = None result = self.jobReport.generateForwardDISET() if not result['OK']: self.log.warn( "Could not generate Operation for job report with result:\n%s" % (result)) else: reportRequest = result['Value'] if reportRequest: self.log.info("Populating request with job report information") self.request.addOperation(reportRequest) accountingReport = None if self.workflow_commons.has_key('AccountingReport'): accountingReport = self.workflow_commons['AccountingReport'] if accountingReport: result = accountingReport.commit() if not result['OK']: self.log.error( "!!! Both accounting and RequestDB are down? !!!") return result if len(self.request): isValid = gRequestValidator.validate(self.request) if not isValid['OK']: raise RuntimeError("Failover request is not valid: %s" % isValid['Message']) else: requestJSON = self.request.toJSON() if requestJSON['OK']: self.log.info( "Creating failover request for deferred operations for job %d" % self.jobID) request_string = str(requestJSON['Value']) self.log.debug(request_string) # Write out the request string fname = '%s_%s_request.json' % (self.production_id, self.prod_job_id) jsonFile = open(fname, 'w') jsonFile.write(request_string) jsonFile.close() self.log.info( "Created file containing failover request %s" % fname) result = self.request.getDigest() if result['OK']: self.log.info("Digest of the request: %s" % result['Value']) else: self.log.error( "No digest? That's not sooo important, anyway: %s" % result['Message']) else: raise RuntimeError(requestJSON['Message'])
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if ( not owner ) or ( not ownerGroup ): res = getProxyInfo( False, False ) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername( owner ) if not res['OK']: return res ownerDN = res['Value'][0] requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass for taskID in sorted( taskDict ): paramDict = taskDict[taskID] if paramDict['InputData']: transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] if type( paramDict['InputData'] ) == type( [] ): files = paramDict['InputData'] elif type( paramDict['InputData'] ) == type( '' ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) oRequest.RequestName = _requestName( transID, taskID ) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = gRequestValidator.validate( oRequest ) if not isValid['OK']: return isValid taskDict[taskID]['TaskObject'] = oRequest return S_OK( taskDict )
def commitRequest( self ): """ Send request to the Request Management Service """ if self.request.isEmpty(): return S_OK() isValid = gRequestValidator.validate( self.request ) if not isValid["OK"]: return S_ERROR( "Failover request is not valid: %s" % isValid["Message"] ) else: requestClient = ReqClient() result = requestClient.putRequest( self.request ) return result
def commitRequest(self): """ Send request to the Request Management Service """ if self.request.isEmpty(): return S_OK() isValid = gRequestValidator.validate(self.request) if not isValid["OK"]: return S_ERROR("Failover request is not valid: %s" % isValid["Message"]) else: requestClient = ReqClient() result = requestClient.putRequest(self.request) return result
def generateFailoverFile( self ): """ Retrieve the accumulated reporting request, and produce a JSON file that is consumed by the JobWrapper """ reportRequest = None result = self.jobReport.generateForwardDISET() if not result['OK']: self.log.warn( "Could not generate Operation for job report with result:\n%s" % ( result ) ) else: reportRequest = result['Value'] if reportRequest: self.log.info( "Populating request with job report information" ) self.request.addOperation( reportRequest ) accountingReport = None if self.workflow_commons.has_key( 'AccountingReport' ): accountingReport = self.workflow_commons['AccountingReport'] if accountingReport: result = accountingReport.commit() if not result['OK']: self.log.error( "!!! Both accounting and RequestDB are down? !!!" ) return result if len( self.request ): isValid = gRequestValidator.validate( self.request ) if not isValid['OK']: raise RuntimeError, "Failover request is not valid: %s" % isValid['Message'] else: requestJSON = self.request.toJSON() if requestJSON['OK']: self.log.info( "Creating failover request for deferred operations for job %d" % self.jobID ) request_string = str( requestJSON['Value'] ) self.log.debug( request_string ) # Write out the request string fname = '%d_%d_request.json' % ( self.production_id, self.prod_job_id ) jsonFile = open( fname, 'w' ) jsonFile.write( request_string ) jsonFile.close() self.log.info( "Created file containing failover request %s" % fname ) result = self.request.getDigest() if result['OK']: self.log.info( "Digest of the request: %s" % result['Value'] ) else: self.log.error( "No digest? That's not sooo important, anyway: %s" % result['Message'] ) else: raise RuntimeError, requestJSON['Message']
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '' ): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass for taskID in sorted( taskDict ): paramDict = taskDict[taskID] if paramDict['InputData']: transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] if type( paramDict['InputData'] ) == type( [] ): files = paramDict['InputData'] elif type( paramDict['InputData'] ) == type( '' ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) oRequest.RequestName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 ) oRequest.OwnerDN = owner oRequest.OwnerGroup = ownerGroup isValid = gRequestValidator.validate( oRequest ) if not isValid['OK']: return isValid taskDict[taskID]['TaskObject'] = oRequest return S_OK( taskDict )
print "Can't get file metadata: %s" % res['Message'] DIRAC.exit( 1 ) if res['Value']['Failed']: print "Could not get the file metadata of the following, so skipping them:" for fFile in res['Value']['Failed']: print fFile lfnMetadata = res['Value']['Successful'] for lfn in lfnMetadata: rarFile = File() rarFile.LFN = lfn rarFile.Size = lfnMetadata[lfn]['Size'] rarFile.Checksum = lfnMetadata[lfn]['Checksum'] rarFile.GUID = lfnMetadata[lfn]['GUID'] rarFile.ChecksumType = 'ADLER32' oOperation.addFile( rarFile ) oRequest.addOperation( oOperation ) isValid = gRequestValidator.validate( oRequest ) if not isValid['OK']: print "Request is not valid: ", isValid['Message'] DIRAC.exit( 1 ) result = reqClient.putRequest( oRequest ) if result['OK']: print 'Request %d Submitted' % result['Value'] else: print 'Failed to submit Request: ', result['Message']
if not res['OK']: print "Can't get file metadata: %s" % res['Message'] DIRAC.exit(1) if res['Value']['Failed']: print "Could not get the file metadata of the following, so skipping them:" for fFile in res['Value']['Failed']: print fFile lfnMetadata = res['Value']['Successful'] for lfn in lfnMetadata: rarFile = File() rarFile.LFN = lfn rarFile.Size = lfnMetadata[lfn]['Size'] rarFile.Checksum = lfnMetadata[lfn]['Checksum'] rarFile.GUID = lfnMetadata[lfn]['GUID'] rarFile.ChecksumType = 'ADLER32' replicateAndRegister.addFile(rarFile) oRequest.addOperation(replicateAndRegister) isValid = gRequestValidator.validate(oRequest) if not isValid['OK']: print "Request is not valid: ", isValid['Message'] DIRAC.exit(1) result = reqClient.putRequest(oRequest) if result['OK']: print "Request %d submitted successfully" % result['Value'] else: print "Failed to submit Request: ", result['Message']