def generateFailoverFile(self):
""" Retrieve the accumulated reporting request, and produce a JSON file that is consumed by the JobWrapper
"""
reportRequest = None
result = self.jobReport.generateForwardDISET()
if not result['OK']:
self.log.warn(
"Could not generate Operation for job report with result:\n%s"
% (result))
else:
reportRequest = result['Value']
if reportRequest:
self.log.info("Populating request with job report information")
self.request.addOperation(reportRequest)
accountingReport = None
if self.workflow_commons.has_key('AccountingReport'):
accountingReport = self.workflow_commons['AccountingReport']
if accountingReport:
result = accountingReport.commit()
if not result['OK']:
self.log.error(
"!!! Both accounting and RequestDB are down? !!!")
return result
if len(self.request):
isValid = gRequestValidator.validate(self.request)
if not isValid['OK']:
raise RuntimeError("Failover request is not valid: %s" %
isValid['Message'])
else:
requestJSON = self.request.toJSON()
if requestJSON['OK']:
self.log.info(
"Creating failover request for deferred operations for job %d"
% self.jobID)
request_string = str(requestJSON['Value'])
self.log.debug(request_string)
# Write out the request string
fname = '%s_%s_request.json' % (self.production_id,
self.prod_job_id)
jsonFile = open(fname, 'w')
jsonFile.write(request_string)
jsonFile.close()
self.log.info(
"Created file containing failover request %s" % fname)
result = self.request.getDigest()
if result['OK']:
self.log.info("Digest of the request: %s" %
result['Value'])
else:
self.log.error(
"No digest? That's not sooo important, anyway: %s"
% result['Message'])
else:
raise RuntimeError(requestJSON['Message'])
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if ( not owner ) or ( not ownerGroup ):
res = getProxyInfo( False, False )
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername( owner )
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if type( paramDict['InputData'] ) == type( [] ):
files = paramDict['InputData']
elif type( paramDict['InputData'] ) == type( '' ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = _requestName( transID, taskID )
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = gRequestValidator.validate( oRequest )
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
def commitRequest( self ):
""" Send request to the Request Management Service
"""
if self.request.isEmpty():
return S_OK()
isValid = gRequestValidator.validate( self.request )
if not isValid["OK"]:
return S_ERROR( "Failover request is not valid: %s" % isValid["Message"] )
else:
requestClient = ReqClient()
result = requestClient.putRequest( self.request )
return result
def commitRequest(self):
""" Send request to the Request Management Service
"""
if self.request.isEmpty():
return S_OK()
isValid = gRequestValidator.validate(self.request)
if not isValid["OK"]:
return S_ERROR("Failover request is not valid: %s" %
isValid["Message"])
else:
requestClient = ReqClient()
result = requestClient.putRequest(self.request)
return result
def generateFailoverFile( self ):
""" Retrieve the accumulated reporting request, and produce a JSON file that is consumed by the JobWrapper
"""
reportRequest = None
result = self.jobReport.generateForwardDISET()
if not result['OK']:
self.log.warn( "Could not generate Operation for job report with result:\n%s" % ( result ) )
else:
reportRequest = result['Value']
if reportRequest:
self.log.info( "Populating request with job report information" )
self.request.addOperation( reportRequest )
accountingReport = None
if self.workflow_commons.has_key( 'AccountingReport' ):
accountingReport = self.workflow_commons['AccountingReport']
if accountingReport:
result = accountingReport.commit()
if not result['OK']:
self.log.error( "!!! Both accounting and RequestDB are down? !!!" )
return result
if len( self.request ):
isValid = gRequestValidator.validate( self.request )
if not isValid['OK']:
raise RuntimeError, "Failover request is not valid: %s" % isValid['Message']
else:
requestJSON = self.request.toJSON()
if requestJSON['OK']:
self.log.info( "Creating failover request for deferred operations for job %d" % self.jobID )
request_string = str( requestJSON['Value'] )
self.log.debug( request_string )
# Write out the request string
fname = '%d_%d_request.json' % ( self.production_id, self.prod_job_id )
jsonFile = open( fname, 'w' )
jsonFile.write( request_string )
jsonFile.close()
self.log.info( "Created file containing failover request %s" % fname )
result = self.request.getDigest()
if result['OK']:
self.log.info( "Digest of the request: %s" % result['Value'] )
else:
self.log.error( "No digest? That's not sooo important, anyway: %s" % result['Message'] )
else:
raise RuntimeError, requestJSON['Message']
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if type( paramDict['InputData'] ) == type( [] ):
files = paramDict['InputData']
elif type( paramDict['InputData'] ) == type( '' ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 )
oRequest.OwnerDN = owner
oRequest.OwnerGroup = ownerGroup
isValid = gRequestValidator.validate( oRequest )
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
print "Can't get file metadata: %s" % res['Message']
DIRAC.exit( 1 )
if res['Value']['Failed']:
print "Could not get the file metadata of the following, so skipping them:"
for fFile in res['Value']['Failed']:
print fFile
lfnMetadata = res['Value']['Successful']
for lfn in lfnMetadata:
rarFile = File()
rarFile.LFN = lfn
rarFile.Size = lfnMetadata[lfn]['Size']
rarFile.Checksum = lfnMetadata[lfn]['Checksum']
rarFile.GUID = lfnMetadata[lfn]['GUID']
rarFile.ChecksumType = 'ADLER32'
oOperation.addFile( rarFile )
oRequest.addOperation( oOperation )
isValid = gRequestValidator.validate( oRequest )
if not isValid['OK']:
print "Request is not valid: ", isValid['Message']
DIRAC.exit( 1 )
result = reqClient.putRequest( oRequest )
if result['OK']:
print 'Request %d Submitted' % result['Value']
else:
print 'Failed to submit Request: ', result['Message']
if not res['OK']:
print "Can't get file metadata: %s" % res['Message']
DIRAC.exit(1)
if res['Value']['Failed']:
print "Could not get the file metadata of the following, so skipping them:"
for fFile in res['Value']['Failed']:
print fFile
lfnMetadata = res['Value']['Successful']
for lfn in lfnMetadata:
rarFile = File()
rarFile.LFN = lfn
rarFile.Size = lfnMetadata[lfn]['Size']
rarFile.Checksum = lfnMetadata[lfn]['Checksum']
rarFile.GUID = lfnMetadata[lfn]['GUID']
rarFile.ChecksumType = 'ADLER32'
replicateAndRegister.addFile(rarFile)
oRequest.addOperation(replicateAndRegister)
isValid = gRequestValidator.validate(oRequest)
if not isValid['OK']:
print "Request is not valid: ", isValid['Message']
DIRAC.exit(1)
result = reqClient.putRequest(oRequest)
if result['OK']:
print "Request %d submitted successfully" % result['Value']
else:
print "Failed to submit Request: ", result['Message']