def __call__(self):
"""It expects to find the arguments for tc.setFileStatusForTransformation in operation.Arguments"""
try:
setFileStatusDict = DEncode.decode(self.operation.Arguments)[0]
self.log.debug("decoded filStatusDict=%s" % str(setFileStatusDict))
except ValueError as error:
self.log.exception(error)
self.operation.Error = str(error)
self.operation.Status = "Failed"
return S_ERROR(str(error))
tc = TransformationClient()
setStatus = tc.setFileStatusForTransformation(
setFileStatusDict["transformation"],
setFileStatusDict["statusDict"], setFileStatusDict["force"])
if not setStatus["OK"]:
errorStr = "failed to change status: %s" % setStatus["Message"]
self.operation.Error = errorStr
self.log.warn(errorStr)
return S_ERROR(self.operation.Error)
else:
self.operation.Status = "Done"
return S_OK()
class FileReport( object ):
""" A stateful object for reporting to TransformationDB
"""
def __init__( self, server = 'Transformation/TransformationManager' ):
""" c'tor
self.transClient is a TransformationClient object
"""
self.transClient = TransformationClient()
self.transClient.setServer( server )
self.statusDict = {}
self.transformation = None
self.force = False
def setFileStatus( self, transformation, lfn, status, sendFlag = False ):
""" Set file status in the context of the given transformation
"""
if not self.transformation:
self.transformation = transformation
self.statusDict[lfn] = status
if sendFlag:
return self.commit()
return S_OK()
def setCommonStatus( self, status ):
""" Set common status for all files in the internal cache
"""
for lfn in self.statusDict.keys():
self.statusDict[lfn] = status
return S_OK()
def getFiles( self ):
""" Get the statuses of the files already accumulated in the FileReport object
"""
return copy.deepcopy( self.statusDict )
def commit( self ):
""" Commit pending file status update records
"""
if not self.statusDict:
return S_OK( {} )
result = self.transClient.setFileStatusForTransformation( self.transformation, self.statusDict, force = self.force )
if result['OK']:
self.statusDict = {}
return result
def generateForwardDISET( self ):
""" Commit the accumulated records and generate request eventually
"""
result = self.commit()
commitOp = None
if not result['OK']:
# Generate Request
commitOp = Operation()
commitOp.Type = 'SetFileStatus'
commitOp.Arguments = DEncode.encode( {'transformation':self.transformation, 'statusDict':self.statusDict, 'force':self.force} )
return S_OK( commitOp )
class FileReport( object ):
""" A stateful object for reporting to TransformationDB
"""
def __init__( self, server = 'Transformation/TransformationManager' ):
self.transClient = TransformationClient()
self.transClient.setServer( server )
self.statusDict = {}
self.transformation = None
self.force = False
def setFileStatus( self, transformation, lfn, status, sendFlag = False ):
""" Set file status in the context of the given transformation """
if not self.transformation:
self.transformation = transformation
self.statusDict[lfn] = status
if sendFlag:
return self.commit()
return S_OK()
def setCommonStatus( self, status ):
""" Set common status for all files in the internal cache """
for lfn in self.statusDict.keys():
self.statusDict[lfn] = status
return S_OK()
def getFiles( self ):
""" Get the statuses of the files already accumulated in the FileReport object """
return copy.deepcopy( self.statusDict )
def commit( self ):
""" Commit pending file status update records """
if not self.statusDict:
return S_OK()
res = self.transClient.setFileStatusForTransformation( self.transformation, self.statusDict, force = self.force )
if not res['OK']:
return res
return S_OK()
def generateForwardDISET( self ):
""" Commit the accumulated records and generate request eventually """
result = self.commit()
forwardDISETOp = None
if not result['OK']:
# Generate Request
if result.has_key( 'rpcStub' ):
forwardDISETOp = Operation()
forwardDISETOp.Type = "ForwardDISET"
forwardDISETOp.Arguments = DEncode.encode( result['rpcStub'] )
else:
return S_ERROR( 'Could not create ForwardDISET operation' )
return S_OK( forwardDISETOp )
class TransformationCLI( cmd.Cmd, API ):
def __init__( self ):
self.server = TransformationClient()
self.indentSpace = 4
cmd.Cmd.__init__( self )
API.__init__( self )
def printPair( self, key, value, separator = ":" ):
valueList = value.split( "\n" )
print "%s%s%s %s" % ( key, " " * ( self.indentSpace - len( key ) ), separator, valueList[0].strip() )
for valueLine in valueList[ 1:-1 ]:
print "%s %s" % ( " " * self.indentSpace, valueLine.strip() )
def do_exit( self, args ):
""" Exits the shell.
usage: exit
"""
sys.exit( 0 )
def do_quit( self, *args ):
""" Exits the shell.
Usage: quit
"""
sys.exit( 0 )
def do_help( self, args ):
""" Default version of the help command
Usage: help <command>
OR use helpall to see description for all commans"""
cmd.Cmd.do_help( self, args )
# overriting default help command
def do_helpall( self, args ):
"""
Shows help information
Usage: helpall <command>
If no command is specified all commands are shown
"""
if len( args ) == 0:
print "\nAvailable commands:\n"
attrList = dir( self )
attrList.sort()
for attribute in attrList:
if attribute.find( "do_" ) == 0:
self.printPair( attribute[ 3: ], getattr( self, attribute ).__doc__[ 1: ] )
print ""
else:
command = args.split()[0].strip()
try:
obj = getattr( self, "do_%s" % command )
except:
print "There's no such %s command" % command
return
self.printPair( command, obj.__doc__[1:] )
def do_shell( self, args ):
"""Execute a shell command
usage !<shell_command>
"""
comm = args
res = shellCall( 0, comm )
if res['OK'] and res['Value'][0] == 0:
returnCode, stdOut, stdErr = res['Value']
print "%s\n%s" % ( stdOut, stdErr )
else:
print res['Message']
def check_params( self, args, num ):
"""Checks if the number of parameters correct"""
argss = string.split( args )
length = len( argss )
if length < num:
print "Error: Number of arguments provided %d less that required %d, please correct." % ( length, num )
return ( False, length )
return ( argss, length )
def check_id_or_name( self, id_or_name ):
"""resolve name or Id by converting type of argument """
if id_or_name.isdigit():
return long( id_or_name ) # its look like id
return id_or_name
def do_setServer( self, args ):
""" Set the destination server
usage: setServer serverURL
"""
argss = string.split( args )
if len( argss ) == 0:
print "no server provided"
self.serverURL = argss[0]
self.server.setServer( self.serverURL )
####################################################################
#
# These are the methods for transformation manipulation
#
def do_getall( self, args ):
"""Get transformation details
usage: getall [Status] [Status]
"""
oTrans = Transformation()
oTrans.setServer( self.serverURL )
oTrans.getTransformations( transStatus = string.split( args ), printOutput = True )
def do_getStatus( self, args ):
"""Get transformation details
usage: getStatus <transName|ID>
"""
argss = string.split( args )
if not len( argss ) > 0:
print "no transformation supplied"
return
for transName in argss:
res = self.server.getTransformation( transName )
if not res['OK']:
print "Getting status of %s failed: %s" % ( transName, res['Message'] )
else:
print "%s: %s" % ( transName, res['Value']['Status'] )
def do_setStatus( self, args ):
"""Set transformation status
usage: setStatus <Status> <transName|ID>
Status <'New' 'Active' 'Stopped' 'Completed' 'Cleaning'>
"""
argss = string.split( args )
if not len( argss ) > 1:
print "transformation and status not supplied"
return
status = argss[0]
transNames = argss[1:]
for transName in transNames:
res = self.server.setTransformationParameter( transName, 'Status', status )
if not res['OK']:
print "Setting status of %s failed: %s" % ( transName, res['Message'] )
else:
print "%s set to %s" % ( transName, status )
def do_start( self, args ):
"""Start transformation
usage: start <transName|ID>
"""
argss = string.split( args )
if not len( argss ) > 0:
print "no transformation supplied"
return
for transName in argss:
res = self.server.setTransformationParameter( transName, 'Status', 'Active' )
if not res['OK']:
print "Setting Status of %s failed: %s" % ( transName, res['Message'] )
else:
res = self.server.setTransformationParameter( transName, 'AgentType', 'Automatic' )
if not res['OK']:
print "Setting AgentType of %s failed: %s" % ( transName, res['Message'] )
else:
print "%s started" % transName
def do_stop( self, args ):
"""Stop transformation
usage: stop <transID|ID>
"""
argss = string.split( args )
if not len( argss ) > 0:
print "no transformation supplied"
return
if not len( argss ) > 0:
print "no transformation supplied"
return
for transName in argss:
res = self.server.setTransformationParameter( transName, 'AgentType', 'Manual' )
if not res['OK']:
print "Stopping of %s failed: %s" % ( transName, res['Message'] )
else:
print "%s stopped" % transName
def do_flush( self, args ):
"""Flush transformation
usage: flush <transName|ID>
"""
argss = string.split( args )
if not len( argss ) > 0:
print "no transformation supplied"
return
for transName in argss:
res = self.server.setTransformationParameter( transName, 'Status', 'Flush' )
if not res['OK']:
print "Flushing of %s failed: %s" % ( transName, res['Message'] )
else:
print "%s flushing" % transName
def do_get( self, args ):
"""Get transformation definition
usage: get <transName|ID>
"""
argss = string.split( args )
if not len( argss ) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformation( transName )
if not res['OK']:
print "Failed to get %s: %s" % ( transName, res['Message'] )
else:
res['Value'].pop( 'Body' )
printDict( res['Value'] )
def do_getBody( self, args ):
"""Get transformation body
usage: getBody <transName|ID>
"""
argss = string.split( args )
if not len( argss ) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformation( transName )
if not res['OK']:
print "Failed to get %s: %s" % ( transName, res['Message'] )
else:
print res['Value']['Body']
def do_getFileStat( self, args ):
"""Get transformation file statistics
usage: getFileStat <transName|ID>
"""
argss = string.split( args )
if not len( argss ) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformationStats( transName )
if not res['OK']:
print "Failed to get statistics for %s: %s" % ( transName, res['Message'] )
else:
res['Value'].pop( 'Total' )
printDict( res['Value'] )
def do_modMask( self, args ):
"""Modify transformation input definition
usage: modInput <mask> <transName|ID>
"""
argss = string.split( args )
if not len( argss ) > 0:
print "no transformation supplied"
return
mask = argss[0]
transNames = argss[1:]
for transName in transNames:
res = self.server.setTransformationParameter( transName, "FileMask", mask )
if not res['OK']:
print "Failed to modify input file mask for %s: %s" % ( transName, res['Message'] )
else:
print "Updated %s filemask" % transName
def do_getFiles( self, args ):
"""Get files for the transformation (optionally with a given status)
usage: getFiles <transName|ID> [Status] [Status]
"""
argss = string.split( args )
if not len( argss ) > 0:
print "no transformation supplied"
return
transName = argss[0]
status = argss[1:]
res = self.server.getTransformation( transName )
if not res['OK']:
print "Failed to get transformation information: %s" % res['Message']
else:
selectDict = {'TransformationID':res['Value']['TransformationID']}
if status:
selectDict['Status'] = status
res = self.server.getTransformationFiles( condDict = selectDict )
if not res['OK']:
print "Failed to get transformation files: %s" % res['Message']
elif res['Value']:
self._printFormattedDictList( res['Value'], ['LFN', 'Status', 'ErrorCount', 'TargetSE', 'LastUpdate'], 'LFN', 'LFN' )
else:
print "No files found"
def do_getFileStatus( self, args ):
"""Get file(s) status for the given transformation
usage: getFileStatus <transName|ID> <lfn> [<lfn>...]
"""
argss = string.split( args )
if len( argss ) < 2:
print "transformation and file not supplied"
return
transName = argss[0]
lfns = argss[1:]
res = self.server.getTransformation( transName )
if not res['OK']:
print "Failed to get transformation information: %s" % res['Message']
else:
selectDict = {'TransformationID':res['Value']['TransformationID']}
res = self.server.getTransformationFiles( condDict = selectDict )
if not res['OK']:
print "Failed to get transformation files: %s" % res['Message']
elif res['Value']:
filesList = []
for fileDict in res['Value']:
if fileDict['LFN'] in lfns:
filesList.append( fileDict )
if filesList:
self._printFormattedDictList( filesList, ['LFN', 'Status', 'ErrorCount', 'TargetSE', 'LastUpdate'], 'LFN', 'LFN' )
else:
print "Could not find any LFN in", lfns, "for transformation", transName
else:
print "No files found"
def do_setFileStatus( self, args ):
"""Set file status for the given transformation
usage: setFileStatus <transName|ID> <lfn> <status>
"""
argss = string.split( args )
if not len( argss ) == 3:
print "transformation file and status not supplied"
return
transName = argss[0]
lfn = argss[1]
status = argss[2]
res = self.server.setFileStatusForTransformation( transName, status, [lfn] )
if not res['OK']:
print "Failed to update file status: %s" % res['Message']
else:
print "Updated file status to %s" % status
def do_resetFile( self, args ):
"""Reset file status for the given transformation
usage: resetFile <transName|ID> <lfn>
"""
argss = string.split( args )
if not len( argss ) > 1:
print "transformation and file(s) not supplied"
return
transName = argss[0]
lfns = argss[1:]
res = self.server.setFileStatusForTransformation( transName, 'Unused', lfns )
if not res['OK']:
print "Failed to reset file status: %s" % res['Message']
else:
if res['Value']['Failed']:
print "Could not reset some files: "
for lfn, reason in res['Value']['Failed'].items():
print lfn, reason
else:
print "Updated file statuses to 'Unused' for %d file(s)" % len( lfns )
def do_resetProcessedFile( self, args ):
""" Reset file status for the given transformation
usage: resetFile <transName|ID> <lfn>
"""
argss = string.split( args )
if not len( argss ) > 1:
print "transformation and file(s) not supplied"
return
transName = argss[0]
lfns = argss[1:]
res = self.server.setFileStatusForTransformation( transName, 'Unused', lfns, force = True )
if not res['OK']:
print "Failed to reset file status: %s" % res['Message']
else:
if res['Value']['Failed']:
print "Could not reset some files: "
for lfn, reason in res['Value']['Failed'].items():
print lfn, reason
else:
print "Updated file statuses to 'Unused' for %d file(s)" % len( lfns )
####################################################################
#
# These are the methods for file manipulation
#
def do_addDirectory( self, args ):
"""Add files from the given catalog directory
usage: addDirectory <directory> [directory]
"""
argss = string.split( args )
if not len( argss ) > 0:
print "no directory supplied"
return
for directory in argss:
res = self.server.addDirectory( directory, force = True )
if not res['OK']:
print 'failed to add directory %s: %s' % ( directory, res['Message'] )
else:
print 'added %s files for %s' % ( res['Value'], directory )
def do_replicas( self, args ):
""" Get replicas for <path>
usage: replicas <lfn> [lfn]
"""
argss = string.split( args )
if not len( argss ) > 0:
print "no files supplied"
return
res = self.server.getReplicas( argss )
if not res['OK']:
print "failed to get any replica information: %s" % res['Message']
return
for lfn in sortList( res['Value']['Failed'].keys() ):
error = res['Value']['Failed'][lfn]
print "failed to get replica information for %s: %s" % ( lfn, error )
for lfn in sortList( res['Value']['Successful'].keys() ):
ses = sortList( res['Value']['Successful'][lfn].keys() )
outStr = "%s :" % lfn.ljust( 100 )
for se in ses:
outStr = "%s %s" % ( outStr, se.ljust( 15 ) )
print outStr
def do_addFile( self, args ):
"""Add new files to transformation DB
usage: addFile <lfn> [lfn]
"""
argss = string.split( args )
if not len( argss ) > 0:
print "no files supplied"
return
lfnDict = {}
for lfn in argss:
lfnDict[lfn] = {'PFN':'IGNORED-PFN', 'SE':'IGNORED-SE', 'Size':0, 'GUID':'IGNORED-GUID', 'Checksum':'IGNORED-CHECKSUM'}
res = self.server.addFile( lfnDict, force = True )
if not res['OK']:
print "failed to add any files: %s" % res['Message']
return
for lfn in sortList( res['Value']['Failed'].keys() ):
error = res['Value']['Failed'][lfn]
print "failed to add %s: %s" % ( lfn, error )
for lfn in sortList( res['Value']['Successful'].keys() ):
print "added %s" % lfn
def do_removeFile( self, args ):
"""Remove file from transformation DB
usage: removeFile <lfn> [lfn]
"""
argss = string.split( args )
if not len( argss ) > 0:
print "no files supplied"
return
res = self.server.removeFile( argss )
if not res['OK']:
print "failed to remove any files: %s" % res['Message']
return
for lfn in sortList( res['Value']['Failed'].keys() ):
error = res['Value']['Failed'][lfn]
print "failed to remove %s: %s" % ( lfn, error )
for lfn in sortList( res['Value']['Successful'].keys() ):
print "removed %s" % lfn
def do_addReplica( self, args ):
""" Add new replica to the transformation DB
usage: addReplica <lfn> <se>
"""
argss = string.split( args )
if not len( argss ) == 2:
print "no file info supplied"
return
lfn = argss[0]
se = argss[1]
lfnDict = {}
lfnDict[lfn] = {'PFN':'IGNORED-PFN', 'SE':se, 'Size':0, 'GUID':'IGNORED-GUID', 'Checksum':'IGNORED-CHECKSUM'}
res = self.server.addReplica( lfnDict, force = True )
if not res['OK']:
print "failed to add replica: %s" % res['Message']
return
for lfn in sortList( res['Value']['Failed'].keys() ):
error = res['Value']['Failed'][lfn]
print "failed to add replica: %s" % ( error )
for lfn in sortList( res['Value']['Successful'].keys() ):
print "added %s" % lfn
def do_removeReplica( self, args ):
"""Remove replica from the transformation DB
usage: removeReplica <lfn> <se>
"""
argss = string.split( args )
if not len( argss ) == 2:
print "no file info supplied"
return
lfn = argss[0]
se = argss[1]
lfnDict = {}
lfnDict[lfn] = {'PFN':'IGNORED-PFN', 'SE':se, 'Size':0, 'GUID':'IGNORED-GUID', 'Checksum':'IGNORED-CHECKSUM'}
res = self.server.removeReplica( lfnDict )
if not res['OK']:
print "failed to remove replica: %s" % res['Message']
return
for lfn in sortList( res['Value']['Failed'].keys() ):
error = res['Value']['Failed'][lfn]
print "failed to remove replica: %s" % ( error )
for lfn in sortList( res['Value']['Successful'].keys() ):
print "removed %s" % lfn
def do_setReplicaStatus( self, args ):
"""Set replica status, usually used to mark a replica Problematic
usage: setReplicaStatus <lfn> <status> <se>
"""
argss = string.split( args )
if not len( argss ) > 2:
print "no file info supplied"
return
lfn = argss[0]
status = argss[1]
se = argss[2]
lfnDict = {}
lfnDict[lfn] = {'Status':status, 'PFN':'IGNORED-PFN', 'SE':se, 'Size':0, 'GUID':'IGNORED-GUID', 'Checksum':'IGNORED-CHECKSUM'}
res = self.server.setReplicaStatus( lfnDict )
if not res['OK']:
print "failed to set replica status: %s" % res['Message']
return
for lfn in sortList( res['Value']['Failed'].keys() ):
error = res['Value']['Failed'][lfn]
print "failed to set replica status: %s" % ( error )
for lfn in sortList( res['Value']['Successful'].keys() ):
print "updated replica status %s" % lfn
userGroup = None
transList = __getTransformations(Script.getPositionalArgs())
if not transList:
DIRAC.exit(1)
requestedLFNs = dmScript.getOption('LFNs')
if not requestedLFNs:
gLogger.always('No files to add')
DIRAC.exit(1)
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
trClient = TransformationClient()
rc = 0
for transID in transList:
res = trClient.setFileStatusForTransformation(transID,
'Removed',
requestedLFNs,
force=True)
if res['OK']:
gLogger.always(
'Successfully set %d files%s Removed in transformation %d' %
(len(res['Value']), (' (out of %d)' % len(requestedLFNs))
if len(res['Value']) != len(requestedLFNs) else '', transID))
else:
gLogger.always(
'Failed to set %d files Removed in transformation %d' %
(len(requestedLFNs), transID), res['Message'])
rc = 2
DIRAC.exit(rc)
class FileReport(object):
"""A stateful object for reporting to TransformationDB"""
def __init__(self, server="Transformation/TransformationManager"):
"""c'tor
self.transClient is a TransformationClient object
"""
self.transClient = TransformationClient()
self.transClient.setServer(server)
self.statusDict = {}
self.transformation = None
self.force = False
def setFileStatus(self, transformation, lfn, status, sendFlag=False):
"""Set file status in the context of the given transformation"""
if not self.transformation:
self.transformation = transformation
if isinstance(lfn, (list, dict, tuple)):
self.statusDict.update(dict.fromkeys(lfn, status))
else:
self.statusDict[lfn] = status
if sendFlag:
return self.commit()
return S_OK()
def setCommonStatus(self, status):
"""Set common status for all files in the internal cache"""
for lfn in self.statusDict:
self.statusDict[lfn] = status
return S_OK()
def getFiles(self):
"""Get the statuses of the files already accumulated in the FileReport object"""
return copy.deepcopy(self.statusDict)
def commit(self):
"""Commit pending file status update records"""
if not self.statusDict:
return S_OK({})
result = self.transClient.setFileStatusForTransformation(
self.transformation, self.statusDict, force=self.force)
if result["OK"]:
self.statusDict = {}
return result
def generateForwardDISET(self):
"""Commit the accumulated records and generate request eventually"""
result = self.commit()
commitOp = None
if not result["OK"]:
# Generate Request
commitOp = Operation()
commitOp.Type = "SetFileStatus"
commitOp.Arguments = DEncode.encode({
"transformation": self.transformation,
"statusDict": self.statusDict,
"force": self.force
})
return S_OK(commitOp)
class FileStatusTransformationAgent(AgentModule):
""" FileStatusTransformationAgent """
def __init__(self, *args, **kwargs):
AgentModule.__init__(self, *args, **kwargs)
self.name = 'FileStatusTransformationAgent'
self.enabled = False
self.shifterProxy = 'DataManager'
self.transformationTypes = ["Replication"]
self.transformationStatuses = ["Active"]
self.transformationFileStatuses = ["Assigned", "Problematic", "Processed", "Unused"]
self.addressTo = ["*****@*****.**"]
self.addressFrom = "*****@*****.**"
self.emailSubject = "FileStatusTransformationAgent"
self.accounting = defaultdict(list)
self.errors = []
self.fcClient = FileCatalogClient()
self.tClient = TransformationClient()
self.reqClient = ReqClient()
self.nClient = NotificationClient()
def checkFileStatusFuncExists(self, status):
""" returns True/False if a function to check transformation files with a given status exists or not """
checkFileStatusFuncName = "check_%s_files" % (status.lower())
if not (hasattr(self, checkFileStatusFuncName) and callable(getattr(self, checkFileStatusFuncName))):
self.log.warn("Unable to process transformation files with status ", status)
return False
return True
def beginExecution(self):
""" Reload the configurations before every cycle """
self.enabled = self.am_getOption('EnableFlag', False)
self.shifterProxy = self.am_setOption('shifterProxy', 'DataManager')
self.transformationTypes = self.am_getOption('TransformationTypes', ["Replication"])
self.transformationStatuses = self.am_getOption('TransformationStatuses', ["Active"])
self.transformationFileStatuses = self.am_getOption(
'TransformationFileStatuses', ["Assigned", "Problematic", "Processed", "Unused"])
self.addressTo = self.am_getOption('MailTo', ["*****@*****.**"])
self.addressFrom = self.am_getOption('MailFrom', "*****@*****.**")
self.transformationFileStatuses = filter(self.checkFileStatusFuncExists, self.transformationFileStatuses)
self.accounting.clear()
return S_OK()
def sendNotification(self, transID, transType=None, sourceSEs=None, targetSEs=None):
""" sends email notification about accounting information of a transformation """
if not(self.errors or self.accounting):
return S_OK()
emailBody = "Transformation ID: %s\n" % transID
if transType:
emailBody += "Transformation Type: %s\n" % transType
if sourceSEs:
emailBody += "Source SE: %s\n" % (" ".join(str(source) for source in sourceSEs))
if targetSEs:
emailBody += "Target SE: %s\n\n" % (" ".join(str(target) for target in targetSEs))
rows = []
for action, transFiles in self.accounting.iteritems():
emailBody += "Total number of files with action %s: %s\n" % (action, len(transFiles))
for transFile in transFiles:
rows.append([[transFile['LFN']], [str(transFile['AvailableOnSource'])],
[str(transFile['AvailableOnTarget'])], [transFile['Status']], [action]])
if rows:
columns = ["LFN", "Source", "Target", "Old Status", "Action"]
emailBody += printTable(columns, rows, printOut=False, numbering=False, columnSeparator=' | ')
if self.errors:
emailBody += "\n\nErrors:"
emailBody += "\n".join(self.errors)
self.log.notice(emailBody)
subject = "%s: %s" % (self.emailSubject, transID)
for address in self.addressTo:
res = self.nClient.sendMail(address, subject, emailBody, self.addressFrom, localAttempt=False)
if not res['OK']:
self.log.error("Failure to send Email notification to ", address)
continue
self.errors = []
self.accounting.clear()
return S_OK()
def logError(self, errStr, varMsg=''):
self.log.error(errStr, varMsg)
self.errors.append(errStr + varMsg)
def execute(self):
""" main execution loop of Agent """
res = self.getTransformations()
if not res['OK']:
self.log.error('Failure to get transformations', res['Message'])
return S_ERROR("Failure to get transformations")
transformations = res['Value']
if not transformations:
self.log.notice('No transformations found with Status %s and Type %s ' %
(self.transformationStatuses, self.transformationTypes))
return S_OK()
self.log.notice('Will treat %d transformations' % len(transformations))
self.log.notice('Transformations: %s' % ",".join([str(transformation['TransformationID'])
for transformation in transformations]))
for trans in transformations:
transID = trans['TransformationID']
if 'SourceSE' not in trans or not trans['SourceSE']:
self.logError("SourceSE not set for transformation, skip processing, transID: ", "%s" % transID)
self.sendNotification(transID)
continue
if 'TargetSE' not in trans or not trans['TargetSE']:
self.logError("TargetSE not set for transformation, skip processing, transID: ", "%s" % transID)
self.sendNotification(transID, sourceSEs=trans['SourceSE'])
continue
if 'DataTransType' not in trans:
self.logError("Transformation Type not set for transformation, skip processing, transID: ", "%s" % transID)
self.sendNotification(transID, sourceSEs=trans['SourceSE'], targetSEs=trans['TargetSE'])
continue
res = self.processTransformation(transID, trans['SourceSE'], trans['TargetSE'], trans['DataTransType'])
if not res['OK']:
self.log.error('Failure to process transformation with ID:', transID)
continue
return S_OK()
def getTransformations(self, transID=None):
""" returns transformations of a given type and status """
res = None
if transID:
res = self.tClient.getTransformations(
condDict={'TransformationID': transID,
'Status': self.transformationStatuses,
'Type': self.transformationTypes})
else:
res = self.tClient.getTransformations(
condDict={'Status': self.transformationStatuses, 'Type': self.transformationTypes})
if not res['OK']:
return res
result = res['Value']
for trans in result:
res = self.tClient.getTransformationParameters(trans['TransformationID'], ['SourceSE', 'TargetSE'])
if not res['OK']:
self.log.error('Failure to get SourceSE and TargetSE parameters for Transformation ID:',
trans['TransformationID'])
continue
trans['SourceSE'] = eval(res['Value']['SourceSE'])
trans['TargetSE'] = eval(res['Value']['TargetSE'])
res = self.getDataTransformationType(trans['TransformationID'])
if not res['OK']:
self.log.error('Failure to determine Data Transformation Type', "%s: %s"
% (trans['TransformationID'], res['Message']))
continue
trans['DataTransType'] = res['Value']
return S_OK(result)
def getRequestStatus(self, transID, taskIDs):
""" returns request statuses for a given list of task IDs """
res = self.tClient.getTransformationTasks(condDict={'TransformationID': transID, 'TaskID': taskIDs})
if not res['OK']:
self.log.error('Failure to get Transformation Tasks for Transformation ID:', transID)
return res
result = res['Value']
requestStatus = {}
for task in result:
requestStatus[task['TaskID']] = {'RequestStatus': task['ExternalStatus'], 'RequestID': long(task['ExternalID'])}
return S_OK(requestStatus)
def getDataTransformationType(self, transID):
""" returns transformation types Replication/Moving/Unknown for a given transformation """
res = self.tClient.getTransformationParameters(transID, 'Body')
if not res['OK']:
return res
# if body is empty then we assume that it is a replication transformation
if not res['Value']:
return S_OK(REPLICATION_TRANS)
replication = False
rmReplica = False
try:
body = json.loads(res['Value'])
for operation in body:
if 'ReplicateAndRegister' in operation:
replication = True
if 'RemoveReplica' in operation:
rmReplica = True
except ValueError:
if 'ReplicateAndRegister' in res['Value']:
replication = True
if 'RemoveReplica' in res['Value']:
rmReplica = True
if rmReplica and replication:
return S_OK(MOVING_TRANS)
if replication:
return S_OK(REPLICATION_TRANS)
return S_ERROR("Unknown Transformation Type '%r'" % res['Value'])
def setFileStatus(self, transID, transFiles, status):
""" sets transformation file status """
lfns = [transFile['LFN'] for transFile in transFiles]
lfnStatuses = {lfn: status for lfn in lfns}
if lfnStatuses:
if self.enabled:
res = self.tClient.setFileStatusForTransformation(transID, newLFNsStatus=lfnStatuses, force=True)
if not res['OK']:
self.logError('Failed to set statuses for LFNs ', "%s" % res['Message'])
return res
for transFile in transFiles:
self.accounting[status].append({'LFN': transFile['LFN'],
'Status': transFile['Status'],
'AvailableOnSource': transFile['AvailableOnSource'],
'AvailableOnTarget': transFile['AvailableOnTarget']})
return S_OK()
def selectFailedRequests(self, transFile):
""" returns True if transformation file has a failed request otherwise returns False """
res = self.getRequestStatus(transFile['TransformationID'], transFile['TaskID'])
if not res['OK']:
self.log.error('Failure to get Request Status for Assigned File')
return False
result = res['Value']
if result[transFile['TaskID']]['RequestStatus'] == 'Failed':
return True
return False
def retryStrategyForFiles(self, transID, transFiles):
""" returns retryStrategy Reset Request if a request is found in RMS, otherwise returns set file status to unused"""
taskIDs = [transFile['TaskID'] for transFile in transFiles]
res = self.getRequestStatus(transID, taskIDs)
if not res['OK']:
return res
result = res['Value']
retryStrategy = defaultdict(dict)
for taskID in taskIDs:
if taskID is None:
self.log.error("Task ID is None", "Transformation: %s\n Files: %r " % (transID, transFiles))
retryStrategy[None]['Strategy'] = SET_UNUSED
continue
res = self.reqClient.getRequest(requestID=result[taskID]['RequestID'])
if not res['OK']:
self.log.notice('Request %s does not exist setting file status to unused' % result[taskID]['RequestID'])
retryStrategy[taskID]['Strategy'] = SET_UNUSED
else:
retryStrategy[taskID]['Strategy'] = SET_UNUSED # RESET_REQUEST
retryStrategy[taskID]['RequestID'] = result[taskID]['RequestID']
return S_OK(retryStrategy)
def check_assigned_files(self, actions, transFiles, transType):
""" treatment for transformation files with assigned status """
for transFile in transFiles:
if transFile['AvailableOnSource'] and transFile['AvailableOnTarget']:
if transType == REPLICATION_TRANS:
actions[SET_PROCESSED].append(transFile)
if transType == MOVING_TRANS:
actions[RETRY].append(transFile)
elif transFile['AvailableOnSource'] and not transFile['AvailableOnTarget']:
actions[RETRY].append(transFile)
elif not transFile['AvailableOnSource'] and transFile['AvailableOnTarget']:
actions[SET_PROCESSED].append(transFile)
else:
# not on src and target
actions[SET_DELETED].append(transFile)
def check_unused_files(self, actions, transFiles, transType):
""" treatment for transformation files with unused status """
for transFile in transFiles:
if not transFile['AvailableOnSource'] and transFile['AvailableOnTarget']:
actions[SET_PROCESSED].append(transFile)
if not transFile['AvailableOnSource'] and not transFile['AvailableOnTarget']:
actions[SET_DELETED].append(transFile)
def check_processed_files(self, actions, transFiles, transType):
""" treatment for transformation files with processed status """
for transFile in transFiles:
if transFile['AvailableOnSource'] and transFile['AvailableOnTarget'] and transType == MOVING_TRANS:
actions[RETRY].append(transFile)
if transFile['AvailableOnSource'] and not transFile['AvailableOnTarget']:
actions[RETRY].append(transFile)
if not transFile['AvailableOnSource'] and not transFile['AvailableOnTarget']:
actions[SET_DELETED].append(transFile)
def check_problematic_files(self, actions, transFiles, transType):
""" treatment for transformation files with problematic status """
for transFile in transFiles:
if transFile['AvailableOnSource'] and transFile['AvailableOnTarget']:
if transType == REPLICATION_TRANS:
actions[SET_PROCESSED].append(transFile)
if transType == MOVING_TRANS:
actions[RETRY].append(transFile)
elif transFile['AvailableOnSource'] and not transFile['AvailableOnTarget']:
actions[RETRY].append(transFile)
elif not transFile['AvailableOnSource'] and transFile['AvailableOnTarget']:
actions[SET_PROCESSED].append(transFile)
else:
# not available on source and target
actions[SET_DELETED].append(transFile)
def retryFiles(self, transID, transFiles):
""" resubmits request or sets file status to unused based on the retry strategy of transformation file """
setFilesUnused = []
setFilesAssigned = []
res = self.retryStrategyForFiles(transID, transFiles)
if not res['OK']:
self.logError('Failure to determine retry strategy (unused / reset request) for files ', "%s" % res['Message'])
return res
retryStrategy = res['Value']
for transFile in transFiles:
if retryStrategy[transFile['TaskID']]['Strategy'] != RESET_REQUEST:
setFilesUnused.append(transFile)
continue
requestID = retryStrategy[transFile['TaskID']]['RequestID']
if self.enabled:
res = self.reqClient.resetFailedRequest(requestID, allR=True)
if not res['OK']:
self.logError('Failed to reset request ', 'ReqID: %s Error: %s' % (requestID, res['Message']))
continue
if res['Value'] == "Not reset":
self.logError('Failed to reset request ', 'ReqID: %s is non-recoverable' % requestID)
continue
setFilesAssigned.append(transFile)
res = self.tClient.setTaskStatus(transID, transFile['TaskID'], 'Waiting')
if not res['OK']:
self.logError('Failure to set Waiting status for Task ID: ', "%s %s" % (transFile['TaskID'], res['Message']))
continue
self.accounting[RESET_REQUEST].append({'LFN': transFile['LFN'],
'Status': transFile['Status'],
'AvailableOnSource': transFile['AvailableOnSource'],
'AvailableOnTarget': transFile['AvailableOnTarget']})
if setFilesUnused:
self.setFileStatus(transID, setFilesUnused, 'Unused')
if setFilesAssigned:
self.setFileStatus(transID, setFilesAssigned, 'Assigned')
return S_OK()
def applyActions(self, transID, actions):
""" sets new file statuses and resets requests """
for action, transFiles in actions.iteritems():
if action == SET_PROCESSED and transFiles:
self.setFileStatus(transID, transFiles, 'Processed')
if action == SET_DELETED and transFiles:
self.setFileStatus(transID, transFiles, 'Deleted')
if action == RETRY and transFiles:
# if there is a request in RMS then reset request otherwise set file status unused
self.retryFiles(transID, transFiles)
def existsInFC(self, storageElements, lfns):
""" checks if files have replicas registered in File Catalog for all given storageElements """
res = self.fcClient.getReplicas(lfns)
if not res['OK']:
return res
result = {}
result['Successful'] = {}
result['Failed'] = {}
setOfSEs = set(storageElements)
for lfn, msg in res['Value']['Failed'].iteritems():
if msg == 'No such file or directory':
result['Successful'][lfn] = False
else:
result['Failed'][lfn] = msg
# check if all replicas are registered in FC
filesFoundInFC = res['Value']['Successful']
for lfn, replicas in filesFoundInFC.iteritems():
result['Successful'][lfn] = setOfSEs.issubset(replicas.keys())
return S_OK(result)
def existsOnSE(self, storageElements, lfns):
""" checks if the given files exist physically on a list of storage elements"""
result = {}
result['Failed'] = {}
result['Successful'] = {}
if not lfns:
return S_OK(result)
voName = lfns[0].split('/')[1]
for se in storageElements:
res = StorageElement(se, vo=voName).exists(lfns)
if not res['OK']:
return res
for lfn, status in res['Value']['Successful'].iteritems():
if lfn not in result['Successful']:
result['Successful'][lfn] = status
if not status:
result['Successful'][lfn] = False
result['Failed'][se] = res['Value']['Failed']
return S_OK(result)
def exists(self, storageElements, lfns):
""" checks if files exists on both file catalog and storage elements """
fcRes = self.existsInFC(storageElements, lfns)
if not fcRes['OK']:
self.logError('Failure to determine if files exists in File Catalog ', "%s" % fcRes['Message'])
return fcRes
if fcRes['Value']['Failed']:
self.logError("Failed FileCatalog Response ", "%s" % fcRes['Value']['Failed'])
# check if files found in file catalog also exist on SE
checkLFNsOnStorage = [lfn for lfn in fcRes['Value']['Successful'] if fcRes['Value']['Successful'][lfn]]
# no files were found in FC, return the result instead of verifying them on SE
if not checkLFNsOnStorage:
return fcRes
seRes = self.existsOnSE(storageElements, checkLFNsOnStorage)
if not seRes['OK']:
self.logError('Failure to determine if files exist on SE ', "%s" % seRes['Message'])
return seRes
for se in storageElements:
if seRes['Value']['Failed'][se]:
self.logError('Failed to determine if files exist on SE ', "%s %s" % (se, seRes['Value']['Failed'][se]))
return S_ERROR()
fcResult = fcRes['Value']['Successful']
seResult = seRes['Value']['Successful']
for lfn in fcResult:
if fcResult[lfn] and not seResult[lfn]:
fcRes['Value']['Successful'][lfn] = False
return fcRes
def processTransformation(self, transID, sourceSE, targetSEs, transType):
""" process transformation for a given transformation ID """
actions = {}
actions[SET_PROCESSED] = []
actions[RETRY] = []
actions[SET_DELETED] = []
for status in self.transformationFileStatuses:
res = self.tClient.getTransformationFiles(condDict={'TransformationID': transID, 'Status': status})
if not res['OK']:
errStr = 'Failure to get Transformation Files, Status: %s Transformation ID: %s Message: %s' % (status,
transID,
res['Message'])
self.logError(errStr)
continue
transFiles = res['Value']
if not transFiles:
self.log.notice("No Transformation Files found with status %s for Transformation ID %d" % (status, transID))
continue
self.log.notice("Processing Transformation Files with status %s for TransformationID %d " % (status, transID))
if status == 'Assigned':
transFiles = filter(self.selectFailedRequests, transFiles)
lfns = [transFile['LFN'] for transFile in transFiles]
if not lfns:
continue
res = self.exists(sourceSE, lfns)
if not res['OK']:
continue
resultSourceSe = res['Value']['Successful']
res = self.exists(targetSEs, lfns)
if not res['OK']:
continue
resultTargetSEs = res['Value']['Successful']
for transFile in transFiles:
lfn = transFile['LFN']
transFile['AvailableOnSource'] = resultSourceSe[lfn]
transFile['AvailableOnTarget'] = resultTargetSEs[lfn]
checkFilesFuncName = "check_%s_files" % status.lower()
checkFiles = getattr(self, checkFilesFuncName)
checkFiles(actions, transFiles, transType)
self.applyActions(transID, actions)
self.sendNotification(transID, transType, sourceSE, targetSEs)
return S_OK()
class FileReport( object ):
""" A stateful object for reporting to TransformationDB
"""
def __init__( self, server = 'Transformation/TransformationManager' ):
self.transClient = TransformationClient()
self.transClient.setServer( server )
self.statusDict = {}
self.transformation = None
def setFileStatus( self, transformation, lfn, status, sendFlag = False ):
""" Set file status in the context of the given transformation """
if not self.transformation:
self.transformation = transformation
self.statusDict[lfn] = status
if sendFlag:
return self.commit()
return S_OK()
def setCommonStatus( self, status ):
""" Set common status for all files in the internal cache """
for lfn in self.statusDict.keys():
self.statusDict[lfn] = status
return S_OK()
def getFiles( self ):
""" Get the statuses of the files already accumulated in the FileReport object """
return copy.deepcopy( self.statusDict )
def commit( self ):
""" Commit pending file status update records """
if not self.statusDict:
return S_OK()
# create intermediate status dictionary
sDict = {}
for lfn, status in self.statusDict.items():
if not sDict.has_key( status ):
sDict[status] = []
sDict[status].append( lfn )
summaryDict = {}
failedResults = []
for status, lfns in sDict.items():
res = self.transClient.setFileStatusForTransformation( self.transformation, status, lfns )
if not res['OK']:
failedResults.append( res )
continue
for lfn, error in res['Value']['Failed'].items():
gLogger.error( "Failed to update file status", "%s %s" % ( lfn, error ) )
if res['Value']['Successful']:
summaryDict[status] = len( res['Value']['Successful'] )
for lfn in res['Value']['Successful']:
self.statusDict.pop( lfn )
if not self.statusDict:
return S_OK( summaryDict )
result = S_ERROR( "Failed to update all file statuses" )
result['FailedResults'] = failedResults
return result
def generateForwardDISET( self ):
""" Commit the accumulated records and generate request eventually """
result = self.commit()
forwardDISETOp = None
if not result['OK']:
# Generate Request
if "FailedResults" in result:
for res in result['FailedResults']:
if 'rpcStub' in res:
forwardDISETOp = Operation()
forwardDISETOp.Type = "ForwardDISET"
forwardDISETOp.Arguments = DEncode.encode( res['rpcStub'] )
return S_OK( forwardDISETOp )
class TransformationCLI(CLI, API):
def __init__(self):
self.server = TransformationClient()
self.indentSpace = 4
CLI.__init__(self)
API.__init__(self)
def printPair(self, key, value, separator=":"):
valueList = value.split("\n")
print "%s%s%s %s" % (key, " " * (self.indentSpace - len(key)),
separator, valueList[0].strip())
for valueLine in valueList[1:-1]:
print "%s %s" % (" " * self.indentSpace, valueLine.strip())
def do_help(self, args):
""" Default version of the help command
Usage: help <command>
OR use helpall to see description for all commands"""
CLI.do_help(self, args)
# overriting default help command
def do_helpall(self, args):
"""
Shows help information
Usage: helpall <command>
If no command is specified all commands are shown
"""
if len(args) == 0:
print "\nAvailable commands:\n"
attrList = dir(self)
attrList.sort()
for attribute in attrList:
if attribute.find("do_") == 0:
self.printPair(attribute[3:],
getattr(self, attribute).__doc__[1:])
print ""
else:
command = args.split()[0].strip()
try:
obj = getattr(self, "do_%s" % command)
except:
print "There's no such %s command" % command
return
self.printPair(command, obj.__doc__[1:])
def do_shell(self, args):
"""Execute a shell command
usage !<shell_command>
"""
comm = args
res = shellCall(0, comm)
if res['OK'] and res['Value'][0] == 0:
_returnCode, stdOut, stdErr = res['Value']
print "%s\n%s" % (stdOut, stdErr)
else:
print res['Message']
def check_params(self, args, num):
"""Checks if the number of parameters correct"""
argss = args.split()
length = len(argss)
if length < num:
print "Error: Number of arguments provided %d less that required %d, please correct." % (
length, num)
return (False, length)
return (argss, length)
def check_id_or_name(self, id_or_name):
"""resolve name or Id by converting type of argument """
if id_or_name.isdigit():
return long(id_or_name) # its look like id
return id_or_name
####################################################################
#
# These are the methods for transformation manipulation
#
def do_getall(self, args):
"""Get transformation details
usage: getall [Status] [Status]
"""
oTrans = Transformation()
oTrans.getTransformations(transStatus=args.split(), printOutput=True)
def do_getAllByUser(self, args):
"""Get all transformations created by a given user
The first argument is the authorDN or username. The authorDN
is preferred: it need to be inside quotes because contains
white spaces. Only authorDN should be quoted.
When the username is provided instead,
the authorDN is retrieved from the uploaded proxy,
so that the retrieved transformations are those created by
the user who uploaded that proxy: that user could be different
that the username provided to the function.
usage: getAllByUser authorDN or username [Status] [Status]
"""
oTrans = Transformation()
argss = args.split()
username = ""
author = ""
status = []
if not len(argss) > 0:
print self.do_getAllByUser.__doc__
return
# if the user didnt quoted the authorDN ends
if '=' in argss[0] and argss[0][0] not in ["'", '"']:
print "AuthorDN need to be quoted (just quote that argument)"
return
if argss[0][0] in ["'", '"']: # authorDN given
author = argss[0]
status_idx = 1
for arg in argss[1:]:
author += ' ' + arg
status_idx += 1
if arg[-1] in ["'", '"']:
break
# At this point we should have something like 'author'
if not author[0] in ["'", '"'] or not author[-1] in ["'", '"']:
print "AuthorDN need to be quoted (just quote that argument)"
return
else:
author = author[1:-1] # throw away the quotes
# the rest are the requested status
status = argss[status_idx:]
else: # username given
username = argss[0]
status = argss[1:]
oTrans.getTransformationsByUser(authorDN=author,
userName=username,
transStatus=status,
printOutput=True)
def do_summaryTransformations(self, args):
"""Show the summary for a list of Transformations
Fields starting with 'F' ('J') refers to files (jobs).
Proc. stand for processed.
Usage: summaryTransformations <ProdID> [<ProdID> ...]
"""
argss = args.split()
if not len(argss) > 0:
print self.do_summaryTransformations.__doc__
return
transid = argss
oTrans = Transformation()
oTrans.getSummaryTransformations(transID=transid)
def do_getStatus(self, args):
"""Get transformation details
usage: getStatus <transName|ID>
"""
argss = args.split()
if not len(argss) > 0:
print "no transformation supplied"
return
for transName in argss:
res = self.server.getTransformation(transName)
if not res['OK']:
print "Getting status of %s failed: %s" % (transName,
res['Message'])
else:
print "%s: %s" % (transName, res['Value']['Status'])
def do_setStatus(self, args):
"""Set transformation status
usage: setStatus <Status> <transName|ID>
Status <'New' 'Active' 'Stopped' 'Completed' 'Cleaning'>
"""
argss = args.split()
if not len(argss) > 1:
print "transformation and status not supplied"
return
status = argss[0]
transNames = argss[1:]
for transName in transNames:
res = self.server.setTransformationParameter(
transName, 'Status', status)
if not res['OK']:
print "Setting status of %s failed: %s" % (transName,
res['Message'])
else:
print "%s set to %s" % (transName, status)
def do_start(self, args):
"""Start transformation
usage: start <transName|ID>
"""
argss = args.split()
if not len(argss) > 0:
print "no transformation supplied"
return
for transName in argss:
res = self.server.setTransformationParameter(
transName, 'Status', 'Active')
if not res['OK']:
print "Setting Status of %s failed: %s" % (transName,
res['Message'])
else:
res = self.server.setTransformationParameter(
transName, 'AgentType', 'Automatic')
if not res['OK']:
print "Setting AgentType of %s failed: %s" % (
transName, res['Message'])
else:
print "%s started" % transName
def do_stop(self, args):
"""Stop transformation
usage: stop <transID|ID>
"""
argss = args.split()
if not len(argss) > 0:
print "no transformation supplied"
return
if not len(argss) > 0:
print "no transformation supplied"
return
for transName in argss:
res = self.server.setTransformationParameter(
transName, 'AgentType', 'Manual')
if not res['OK']:
print "Stopping of %s failed: %s" % (transName, res['Message'])
else:
print "%s stopped" % transName
def do_flush(self, args):
"""Flush transformation
usage: flush <transName|ID>
"""
argss = args.split()
if not len(argss) > 0:
print "no transformation supplied"
return
for transName in argss:
res = self.server.setTransformationParameter(
transName, 'Status', 'Flush')
if not res['OK']:
print "Flushing of %s failed: %s" % (transName, res['Message'])
else:
print "%s flushing" % transName
def do_get(self, args):
"""Get transformation definition
usage: get <transName|ID>
"""
argss = args.split()
if not len(argss) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformation(transName)
if not res['OK']:
print "Failed to get %s: %s" % (transName, res['Message'])
else:
res['Value'].pop('Body')
printDict(res['Value'])
def do_getBody(self, args):
"""Get transformation body
usage: getBody <transName|ID>
"""
argss = args.split()
if not len(argss) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformation(transName)
if not res['OK']:
print "Failed to get %s: %s" % (transName, res['Message'])
else:
print res['Value']['Body']
def do_getFileStat(self, args):
"""Get transformation file statistics
usage: getFileStat <transName|ID>
"""
argss = args.split()
if not len(argss) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformationStats(transName)
if not res['OK']:
print "Failed to get statistics for %s: %s" % (transName,
res['Message'])
else:
res['Value'].pop('Total')
printDict(res['Value'])
def do_modMask(self, args):
"""Modify transformation input definition
usage: modInput <mask> <transName|ID>
"""
argss = args.split()
if not len(argss) > 0:
print "no transformation supplied"
return
mask = argss[0]
transNames = argss[1:]
for transName in transNames:
res = self.server.setTransformationParameter(
transName, "FileMask", mask)
if not res['OK']:
print "Failed to modify input file mask for %s: %s" % (
transName, res['Message'])
else:
print "Updated %s filemask" % transName
def do_getFiles(self, args):
"""Get files for the transformation (optionally with a given status)
usage: getFiles <transName|ID> [Status] [Status]
"""
argss = args.split()
if not len(argss) > 0:
print "no transformation supplied"
return
transName = argss[0]
status = argss[1:]
res = self.server.getTransformation(transName)
if not res['OK']:
print "Failed to get transformation information: %s" % res[
'Message']
else:
selectDict = {'TransformationID': res['Value']['TransformationID']}
if status:
selectDict['Status'] = status
res = self.server.getTransformationFiles(condDict=selectDict)
if not res['OK']:
print "Failed to get transformation files: %s" % res['Message']
elif res['Value']:
self._printFormattedDictList(
res['Value'],
['LFN', 'Status', 'ErrorCount', 'TargetSE', 'LastUpdate'],
'LFN', 'LFN')
else:
print "No files found"
def do_getFileStatus(self, args):
"""Get file(s) status for the given transformation
usage: getFileStatus <transName|ID> <lfn> [<lfn>...]
"""
argss = args.split()
if len(argss) < 2:
print "transformation and file not supplied"
return
transName = argss[0]
lfns = argss[1:]
res = self.server.getTransformation(transName)
if not res['OK']:
print "Failed to get transformation information: %s" % res[
'Message']
else:
selectDict = {'TransformationID': res['Value']['TransformationID']}
res = self.server.getTransformationFiles(condDict=selectDict)
if not res['OK']:
print "Failed to get transformation files: %s" % res['Message']
elif res['Value']:
filesList = []
for fileDict in res['Value']:
if fileDict['LFN'] in lfns:
filesList.append(fileDict)
if filesList:
self._printFormattedDictList(filesList, [
'LFN', 'Status', 'ErrorCount', 'TargetSE', 'LastUpdate'
], 'LFN', 'LFN')
else:
print "Could not find any LFN in", lfns, "for transformation", transName
else:
print "No files found"
def do_getOutputFiles(self, args):
"""Get output files for the transformation
usage: getOutputFiles <transName|ID>
"""
argss = args.split()
if not len(argss) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformation(transName)
if not res['OK']:
print "Failed to get transformation information: %s" % res[
'Message']
else:
fc = FileCatalog()
meta = {}
meta['ProdID'] = transName
res = fc.findFilesByMetadata(meta)
if not res['OK']:
print res['Message']
return
if not len(res['Value']) > 0:
print 'No output files yet for transformation %d' % int(
transName)
return
else:
for lfn in res['Value']:
print lfn
def do_getInputDataQuery(self, args):
"""Get input data query for the transformation
usage: getInputDataQuery <transName|ID>
"""
argss = args.split()
if not len(argss) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformationInputDataQuery(transName)
if not res['OK']:
print "Failed to get transformation input data query: %s" % res[
'Message']
else:
print res['Value']
def do_setFileStatus(self, args):
"""Set file status for the given transformation
usage: setFileStatus <transName|ID> <lfn> <status>
"""
argss = args.split()
if not len(argss) == 3:
print "transformation file and status not supplied"
return
transName = argss[0]
lfn = argss[1]
status = argss[2]
res = self.server.setFileStatusForTransformation(
transName, status, [lfn])
if not res['OK']:
print "Failed to update file status: %s" % res['Message']
else:
print "Updated file status to %s" % status
def do_resetFile(self, args):
"""Reset file status for the given transformation
usage: resetFile <transName|ID> <lfns>
"""
argss = args.split()
if not len(argss) > 1:
print "transformation and file(s) not supplied"
return
transName = argss[0]
lfns = argss[1:]
res = self.server.setFileStatusForTransformation(
transName, 'Unused', lfns)
if not res['OK']:
print "Failed to reset file status: %s" % res['Message']
else:
if 'Failed' in res['Value']:
print "Could not reset some files: "
for lfn, reason in res['Value']['Failed'].items():
print lfn, reason
else:
print "Updated file statuses to 'Unused' for %d file(s)" % len(
lfns)
def do_resetProcessedFile(self, args):
""" Reset file status for the given transformation
usage: resetFile <transName|ID> <lfn>
"""
argss = args.split()
if not len(argss) > 1:
print "transformation and file(s) not supplied"
return
transName = argss[0]
lfns = argss[1:]
res = self.server.setFileStatusForTransformation(transName,
'Unused',
lfns,
force=True)
if not res['OK']:
print "Failed to reset file status: %s" % res['Message']
else:
if 'Failed' in res['Value'] and res['Value']['Failed']:
print "Could not reset some files: "
for lfn, reason in res['Value']['Failed'].items():
print lfn, reason
else:
print "Updated file statuses to 'Unused' for %d file(s)" % len(
lfns)
####################################################################
#
# These are the methods for file manipulation
#
def do_addDirectory(self, args):
"""Add files from the given catalog directory
usage: addDirectory <directory> [directory]
"""
argss = args.split()
if not len(argss) > 0:
print "no directory supplied"
return
for directory in argss:
res = self.server.addDirectory(directory, force=True)
if not res['OK']:
print 'failed to add directory %s: %s' % (directory,
res['Message'])
else:
print 'added %s files for %s' % (res['Value'], directory)
def do_replicas(self, args):
""" Get replicas for <path>
usage: replicas <lfn> [lfn]
"""
argss = args.split()
if not len(argss) > 0:
print "no files supplied"
return
res = self.server.getReplicas(argss)
if not res['OK']:
print "failed to get any replica information: %s" % res['Message']
return
for lfn in sorted(res['Value']['Failed'].keys()):
error = res['Value']['Failed'][lfn]
print "failed to get replica information for %s: %s" % (lfn, error)
for lfn in sorted(res['Value']['Successful'].keys()):
ses = sorted(res['Value']['Successful'][lfn].keys())
outStr = "%s :" % lfn.ljust(100)
for se in ses:
outStr = "%s %s" % (outStr, se.ljust(15))
print outStr
def do_addFile(self, args):
"""Add new files to transformation DB
usage: addFile <lfn> [lfn]
"""
argss = args.split()
if not len(argss) > 0:
print "no files supplied"
return
lfnDict = {}
for lfn in argss:
lfnDict[lfn] = {
'PFN': 'IGNORED-PFN',
'SE': 'IGNORED-SE',
'Size': 0,
'GUID': 'IGNORED-GUID',
'Checksum': 'IGNORED-CHECKSUM'
}
res = self.server.addFile(lfnDict, force=True)
if not res['OK']:
print "failed to add any files: %s" % res['Message']
return
for lfn in sorted(res['Value']['Failed'].keys()):
error = res['Value']['Failed'][lfn]
print "failed to add %s: %s" % (lfn, error)
for lfn in sorted(res['Value']['Successful'].keys()):
print "added %s" % lfn
def do_removeFile(self, args):
"""Remove file from transformation DB
usage: removeFile <lfn> [lfn]
"""
argss = args.split()
if not len(argss) > 0:
print "no files supplied"
return
res = self.server.removeFile(argss)
if not res['OK']:
print "failed to remove any files: %s" % res['Message']
return
for lfn in sorted(res['Value']['Failed'].keys()):
error = res['Value']['Failed'][lfn]
print "failed to remove %s: %s" % (lfn, error)
for lfn in sorted(res['Value']['Successful'].keys()):
print "removed %s" % lfn
def do_addReplica(self, args):
""" Add new replica to the transformation DB
usage: addReplica <lfn> <se>
"""
argss = args.split()
if not len(argss) == 2:
print "no file info supplied"
return
lfn = argss[0]
se = argss[1]
lfnDict = {}
lfnDict[lfn] = {
'PFN': 'IGNORED-PFN',
'SE': se,
'Size': 0,
'GUID': 'IGNORED-GUID',
'Checksum': 'IGNORED-CHECKSUM'
}
res = self.server.addReplica(lfnDict, force=True)
if not res['OK']:
print "failed to add replica: %s" % res['Message']
return
for lfn in sorted(res['Value']['Failed'].keys()):
error = res['Value']['Failed'][lfn]
print "failed to add replica: %s" % (error)
for lfn in sorted(res['Value']['Successful'].keys()):
print "added %s" % lfn
def do_removeReplica(self, args):
"""Remove replica from the transformation DB
usage: removeReplica <lfn> <se>
"""
argss = args.split()
if not len(argss) == 2:
print "no file info supplied"
return
lfn = argss[0]
se = argss[1]
lfnDict = {}
lfnDict[lfn] = {
'PFN': 'IGNORED-PFN',
'SE': se,
'Size': 0,
'GUID': 'IGNORED-GUID',
'Checksum': 'IGNORED-CHECKSUM'
}
res = self.server.removeReplica(lfnDict)
if not res['OK']:
print "failed to remove replica: %s" % res['Message']
return
for lfn in sorted(res['Value']['Failed'].keys()):
error = res['Value']['Failed'][lfn]
print "failed to remove replica: %s" % (error)
for lfn in sorted(res['Value']['Successful'].keys()):
print "removed %s" % lfn
def do_setReplicaStatus(self, args):
"""Set replica status, usually used to mark a replica Problematic
usage: setReplicaStatus <lfn> <status> <se>
"""
argss = args.split()
if not len(argss) > 2:
print "no file info supplied"
return
lfn = argss[0]
status = argss[1]
se = argss[2]
lfnDict = {}
lfnDict[lfn] = {
'Status': status,
'PFN': 'IGNORED-PFN',
'SE': se,
'Size': 0,
'GUID': 'IGNORED-GUID',
'Checksum': 'IGNORED-CHECKSUM'
}
res = self.server.setReplicaStatus(lfnDict)
if not res['OK']:
print "failed to set replica status: %s" % res['Message']
return
for lfn in sorted(res['Value']['Failed'].keys()):
error = res['Value']['Failed'][lfn]
print "failed to set replica status: %s" % (error)
for lfn in sorted(res['Value']['Successful'].keys()):
print "updated replica status %s" % lfn
class FileReport:
''' A stateful object for reporting to TransformationDB
'''
def __init__(self, server='Transformation/TransformationManager'):
self.client = TransformationClient()
self.client.setServer(server)
self.statusDict = {}
self.transformation = None
def setFileStatus(self, transformation, lfn, status, sendFlag=False):
''' Set file status in the context of the given transformation '''
if not self.transformation:
self.transformation = transformation
self.statusDict[lfn] = status
if sendFlag:
return self.commit()
return S_OK()
def setCommonStatus(self, status):
''' Set common status for all files in the internal cache '''
for lfn in self.statusDict.keys():
self.statusDict[lfn] = status
return S_OK()
def getFiles(self):
''' Get the statuses of the files already accumulated in the FileReport object '''
return copy.deepcopy(self.statusDict)
def commit(self):
''' Commit pending file status update records '''
if not self.statusDict:
return S_OK()
# create intermediate status dictionary
sDict = {}
for lfn, status in self.statusDict.items():
if not sDict.has_key(status):
sDict[status] = []
sDict[status].append(lfn)
summaryDict = {}
failedResults = []
for status, lfns in sDict.items():
res = self.client.setFileStatusForTransformation(
self.transformation, status, lfns)
if not res['OK']:
failedResults.append(res)
continue
for lfn, error in res['Value']['Failed'].items():
gLogger.error("Failed to update file status",
"%s %s" % (lfn, error))
if res['Value']['Successful']:
summaryDict[status] = len(res['Value']['Successful'])
for lfn in res['Value']['Successful']:
self.statusDict.pop(lfn)
if not self.statusDict:
return S_OK(summaryDict)
result = S_ERROR("Failed to update all file statuses")
result['FailedResults'] = failedResults
return result
def generateRequest(self):
''' Commit the accumulated records and generate request eventually '''
result = self.commit()
request = None
if not result['OK']:
# Generate Request
request = RequestContainer()
if result.has_key('FailedResults'):
for res in result['FailedResults']:
if res.has_key('rpcStub'):
request.setDISETRequest(res['rpcStub'])
return S_OK(request)
class TransformationCLI( cmd.Cmd, API ):
def __init__( self ):
self.server = TransformationClient()
self.indentSpace = 4
cmd.Cmd.__init__( self )
API.__init__( self )
def printPair( self, key, value, separator = ":" ):
valueList = value.split( "\n" )
print "%s%s%s %s" % ( key, " " * ( self.indentSpace - len( key ) ), separator, valueList[0].strip() )
for valueLine in valueList[ 1:-1 ]:
print "%s %s" % ( " " * self.indentSpace, valueLine.strip() )
def do_exit( self, args ):
""" Exits the shell.
usage: exit
"""
sys.exit( 0 )
def do_quit( self, *args ):
""" Exits the shell.
Usage: quit
"""
sys.exit( 0 )
def do_help( self, args ):
""" Default version of the help command
Usage: help <command>
OR use helpall to see description for all commands"""
cmd.Cmd.do_help( self, args )
# overriting default help command
def do_helpall( self, args ):
"""
Shows help information
Usage: helpall <command>
If no command is specified all commands are shown
"""
if len( args ) == 0:
print "\nAvailable commands:\n"
attrList = dir( self )
attrList.sort()
for attribute in attrList:
if attribute.find( "do_" ) == 0:
self.printPair( attribute[ 3: ], getattr( self, attribute ).__doc__[ 1: ] )
print ""
else:
command = args.split()[0].strip()
try:
obj = getattr( self, "do_%s" % command )
except:
print "There's no such %s command" % command
return
self.printPair( command, obj.__doc__[1:] )
def do_shell( self, args ):
"""Execute a shell command
usage !<shell_command>
"""
comm = args
res = shellCall( 0, comm )
if res['OK'] and res['Value'][0] == 0:
_returnCode, stdOut, stdErr = res['Value']
print "%s\n%s" % ( stdOut, stdErr )
else:
print res['Message']
def check_params( self, args, num ):
"""Checks if the number of parameters correct"""
argss = args.split()
length = len( argss )
if length < num:
print "Error: Number of arguments provided %d less that required %d, please correct." % ( length, num )
return ( False, length )
return ( argss, length )
def check_id_or_name( self, id_or_name ):
"""resolve name or Id by converting type of argument """
if id_or_name.isdigit():
return long( id_or_name ) # its look like id
return id_or_name
####################################################################
#
# These are the methods for transformation manipulation
#
def do_getall( self, args ):
"""Get transformation details
usage: getall [Status] [Status]
"""
oTrans = Transformation()
oTrans.getTransformations( transStatus = args.split(), printOutput = True )
def do_getAllByUser( self, args ):
"""Get all transformations created by a given user
The first argument is the authorDN or username. The authorDN
is preferred: it need to be inside quotes because contains
white spaces. Only authorDN should be quoted.
When the username is provided instead,
the authorDN is retrieved from the uploaded proxy,
so that the retrieved transformations are those created by
the user who uploaded that proxy: that user could be different
that the username provided to the function.
usage: getAllByUser authorDN or username [Status] [Status]
"""
oTrans = Transformation()
argss = args.split()
username = ""
author = ""
status = []
if not len( argss ) > 0:
print self.do_getAllByUser.__doc__
return
# if the user didnt quoted the authorDN ends
if '=' in argss[0] and argss[0][0] not in ["'", '"']:
print "AuthorDN need to be quoted (just quote that argument)"
return
if argss[0][0] in ["'", '"']: # authorDN given
author = argss[0]
status_idx = 1
for arg in argss[1:]:
author += ' ' + arg
status_idx +=1
if arg[-1] in ["'", '"']:
break
# At this point we should have something like 'author'
if not author[0] in ["'", '"'] or not author[-1] in ["'", '"']:
print "AuthorDN need to be quoted (just quote that argument)"
return
else:
author = author[1:-1] # throw away the quotes
# the rest are the requested status
status = argss[ status_idx: ]
else: # username given
username = argss[0]
status = argss[ 1: ]
oTrans.getTransformationsByUser( authorDN = author, userName = username, transStatus = status, printOutput = True )
def do_summaryTransformations( self, args ):
"""Show the summary for a list of Transformations
Fields starting with 'F' ('J') refers to files (jobs).
Proc. stand for processed.
Usage: summaryTransformations <ProdID> [<ProdID> ...]
"""
argss = args.split()
if not len( argss ) > 0:
print self.do_summaryTransformations.__doc__
return
transid = argss
oTrans = Transformation()
oTrans.getSummaryTransformations( transID = transid )
def do_getStatus( self, args ):
"""Get transformation details
usage: getStatus <transName|ID>
"""
argss = args.split()
if not len( argss ) > 0:
print "no transformation supplied"
return
for transName in argss:
res = self.server.getTransformation( transName )
if not res['OK']:
print "Getting status of %s failed: %s" % ( transName, res['Message'] )
else:
print "%s: %s" % ( transName, res['Value']['Status'] )
def do_setStatus( self, args ):
"""Set transformation status
usage: setStatus <Status> <transName|ID>
Status <'New' 'Active' 'Stopped' 'Completed' 'Cleaning'>
"""
argss = args.split()
if not len( argss ) > 1:
print "transformation and status not supplied"
return
status = argss[0]
transNames = argss[1:]
for transName in transNames:
res = self.server.setTransformationParameter( transName, 'Status', status )
if not res['OK']:
print "Setting status of %s failed: %s" % ( transName, res['Message'] )
else:
print "%s set to %s" % ( transName, status )
def do_start( self, args ):
"""Start transformation
usage: start <transName|ID>
"""
argss = args.split()
if not len( argss ) > 0:
print "no transformation supplied"
return
for transName in argss:
res = self.server.setTransformationParameter( transName, 'Status', 'Active' )
if not res['OK']:
print "Setting Status of %s failed: %s" % ( transName, res['Message'] )
else:
res = self.server.setTransformationParameter( transName, 'AgentType', 'Automatic' )
if not res['OK']:
print "Setting AgentType of %s failed: %s" % ( transName, res['Message'] )
else:
print "%s started" % transName
def do_stop( self, args ):
"""Stop transformation
usage: stop <transID|ID>
"""
argss = args.split()
if not len( argss ) > 0:
print "no transformation supplied"
return
if not len( argss ) > 0:
print "no transformation supplied"
return
for transName in argss:
res = self.server.setTransformationParameter( transName, 'AgentType', 'Manual' )
if not res['OK']:
print "Stopping of %s failed: %s" % ( transName, res['Message'] )
else:
print "%s stopped" % transName
def do_flush( self, args ):
"""Flush transformation
usage: flush <transName|ID>
"""
argss = args.split()
if not len( argss ) > 0:
print "no transformation supplied"
return
for transName in argss:
res = self.server.setTransformationParameter( transName, 'Status', 'Flush' )
if not res['OK']:
print "Flushing of %s failed: %s" % ( transName, res['Message'] )
else:
print "%s flushing" % transName
def do_get( self, args ):
"""Get transformation definition
usage: get <transName|ID>
"""
argss = args.split()
if not len( argss ) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformation( transName )
if not res['OK']:
print "Failed to get %s: %s" % ( transName, res['Message'] )
else:
res['Value'].pop( 'Body' )
printDict( res['Value'] )
def do_getBody( self, args ):
"""Get transformation body
usage: getBody <transName|ID>
"""
argss = args.split()
if not len( argss ) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformation( transName )
if not res['OK']:
print "Failed to get %s: %s" % ( transName, res['Message'] )
else:
print res['Value']['Body']
def do_getFileStat( self, args ):
"""Get transformation file statistics
usage: getFileStat <transName|ID>
"""
argss = args.split()
if not len( argss ) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformationStats( transName )
if not res['OK']:
print "Failed to get statistics for %s: %s" % ( transName, res['Message'] )
else:
res['Value'].pop( 'Total' )
printDict( res['Value'] )
def do_modMask( self, args ):
"""Modify transformation input definition
usage: modInput <mask> <transName|ID>
"""
argss = args.split()
if not len( argss ) > 0:
print "no transformation supplied"
return
mask = argss[0]
transNames = argss[1:]
for transName in transNames:
res = self.server.setTransformationParameter( transName, "FileMask", mask )
if not res['OK']:
print "Failed to modify input file mask for %s: %s" % ( transName, res['Message'] )
else:
print "Updated %s filemask" % transName
def do_getFiles( self, args ):
"""Get files for the transformation (optionally with a given status)
usage: getFiles <transName|ID> [Status] [Status]
"""
argss = args.split()
if not len( argss ) > 0:
print "no transformation supplied"
return
transName = argss[0]
status = argss[1:]
res = self.server.getTransformation( transName )
if not res['OK']:
print "Failed to get transformation information: %s" % res['Message']
else:
selectDict = {'TransformationID':res['Value']['TransformationID']}
if status:
selectDict['Status'] = status
res = self.server.getTransformationFiles( condDict = selectDict )
if not res['OK']:
print "Failed to get transformation files: %s" % res['Message']
elif res['Value']:
self._printFormattedDictList( res['Value'], ['LFN', 'Status', 'ErrorCount', 'TargetSE', 'LastUpdate'],
'LFN', 'LFN' )
else:
print "No files found"
def do_getFileStatus( self, args ):
"""Get file(s) status for the given transformation
usage: getFileStatus <transName|ID> <lfn> [<lfn>...]
"""
argss = args.split()
if len( argss ) < 2:
print "transformation and file not supplied"
return
transName = argss[0]
lfns = argss[1:]
res = self.server.getTransformation( transName )
if not res['OK']:
print "Failed to get transformation information: %s" % res['Message']
else:
selectDict = {'TransformationID':res['Value']['TransformationID']}
res = self.server.getTransformationFiles( condDict = selectDict )
if not res['OK']:
print "Failed to get transformation files: %s" % res['Message']
elif res['Value']:
filesList = []
for fileDict in res['Value']:
if fileDict['LFN'] in lfns:
filesList.append( fileDict )
if filesList:
self._printFormattedDictList( filesList, ['LFN', 'Status', 'ErrorCount', 'TargetSE', 'LastUpdate'],
'LFN', 'LFN' )
else:
print "Could not find any LFN in", lfns, "for transformation", transName
else:
print "No files found"
def do_getOutputFiles( self, args ):
"""Get output files for the transformation
usage: getOutputFiles <transName|ID>
"""
argss = args.split()
if not len( argss ) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformation( transName )
if not res['OK']:
print "Failed to get transformation information: %s" % res['Message']
else:
fc = FileCatalog()
meta = {}
meta ['ProdID'] = transName
res = fc.findFilesByMetadata( meta )
if not res['OK']:
print res['Message']
return
if not len( res['Value'] ) > 0:
print 'No output files yet for transformation %d' %int(transName)
return
else:
for lfn in res['Value']:
print lfn
def do_getInputDataQuery( self, args ):
"""Get input data query for the transformation
usage: getInputDataQuery <transName|ID>
"""
argss = args.split()
if not len( argss ) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformationInputDataQuery( transName )
if not res['OK']:
print "Failed to get transformation input data query: %s" % res['Message']
else:
print res['Value']
def do_setFileStatus( self, args ):
"""Set file status for the given transformation
usage: setFileStatus <transName|ID> <lfn> <status>
"""
argss = args.split()
if not len( argss ) == 3:
print "transformation file and status not supplied"
return
transName = argss[0]
lfn = argss[1]
status = argss[2]
res = self.server.setFileStatusForTransformation( transName, status, [lfn] )
if not res['OK']:
print "Failed to update file status: %s" % res['Message']
else:
print "Updated file status to %s" % status
def do_resetFile( self, args ):
"""Reset file status for the given transformation
usage: resetFile <transName|ID> <lfns>
"""
argss = args.split()
if not len( argss ) > 1:
print "transformation and file(s) not supplied"
return
transName = argss[0]
lfns = argss[1:]
res = self.server.setFileStatusForTransformation( transName, 'Unused', lfns )
if not res['OK']:
print "Failed to reset file status: %s" % res['Message']
else:
if 'Failed' in res['Value']:
print "Could not reset some files: "
for lfn, reason in res['Value']['Failed'].items():
print lfn, reason
else:
print "Updated file statuses to 'Unused' for %d file(s)" % len( lfns )
def do_resetProcessedFile( self, args ):
""" Reset file status for the given transformation
usage: resetFile <transName|ID> <lfn>
"""
argss = args.split()
if not len( argss ) > 1:
print "transformation and file(s) not supplied"
return
transName = argss[0]
lfns = argss[1:]
res = self.server.setFileStatusForTransformation( transName, 'Unused', lfns, force = True )
if not res['OK']:
print "Failed to reset file status: %s" % res['Message']
else:
if res['Value']['Failed']:
print "Could not reset some files: "
for lfn, reason in res['Value']['Failed'].items():
print lfn, reason
else:
print "Updated file statuses to 'Unused' for %d file(s)" % len( lfns )
####################################################################
#
# These are the methods for file manipulation
#
def do_addDirectory( self, args ):
"""Add files from the given catalog directory
usage: addDirectory <directory> [directory]
"""
argss = args.split()
if not len( argss ) > 0:
print "no directory supplied"
return
for directory in argss:
res = self.server.addDirectory( directory, force = True )
if not res['OK']:
print 'failed to add directory %s: %s' % ( directory, res['Message'] )
else:
print 'added %s files for %s' % ( res['Value'], directory )
def do_replicas( self, args ):
""" Get replicas for <path>
usage: replicas <lfn> [lfn]
"""
argss = args.split()
if not len( argss ) > 0:
print "no files supplied"
return
res = self.server.getReplicas( argss )
if not res['OK']:
print "failed to get any replica information: %s" % res['Message']
return
for lfn in sorted( res['Value']['Failed'].keys() ):
error = res['Value']['Failed'][lfn]
print "failed to get replica information for %s: %s" % ( lfn, error )
for lfn in sorted( res['Value']['Successful'].keys() ):
ses = sorted( res['Value']['Successful'][lfn].keys() )
outStr = "%s :" % lfn.ljust( 100 )
for se in ses:
outStr = "%s %s" % ( outStr, se.ljust( 15 ) )
print outStr
def do_addFile( self, args ):
"""Add new files to transformation DB
usage: addFile <lfn> [lfn]
"""
argss = args.split()
if not len( argss ) > 0:
print "no files supplied"
return
lfnDict = {}
for lfn in argss:
lfnDict[lfn] = {'PFN':'IGNORED-PFN', 'SE':'IGNORED-SE', 'Size':0, 'GUID':'IGNORED-GUID',
'Checksum':'IGNORED-CHECKSUM'}
res = self.server.addFile( lfnDict, force = True )
if not res['OK']:
print "failed to add any files: %s" % res['Message']
return
for lfn in sorted( res['Value']['Failed'].keys() ):
error = res['Value']['Failed'][lfn]
print "failed to add %s: %s" % ( lfn, error )
for lfn in sorted( res['Value']['Successful'].keys() ):
print "added %s" % lfn
def do_removeFile( self, args ):
"""Remove file from transformation DB
usage: removeFile <lfn> [lfn]
"""
argss = args.split()
if not len( argss ) > 0:
print "no files supplied"
return
res = self.server.removeFile( argss )
if not res['OK']:
print "failed to remove any files: %s" % res['Message']
return
for lfn in sorted( res['Value']['Failed'].keys() ):
error = res['Value']['Failed'][lfn]
print "failed to remove %s: %s" % ( lfn, error )
for lfn in sorted( res['Value']['Successful'].keys() ):
print "removed %s" % lfn
def do_addReplica( self, args ):
""" Add new replica to the transformation DB
usage: addReplica <lfn> <se>
"""
argss = args.split()
if not len( argss ) == 2:
print "no file info supplied"
return
lfn = argss[0]
se = argss[1]
lfnDict = {}
lfnDict[lfn] = {'PFN':'IGNORED-PFN', 'SE':se, 'Size':0, 'GUID':'IGNORED-GUID', 'Checksum':'IGNORED-CHECKSUM'}
res = self.server.addReplica( lfnDict, force = True )
if not res['OK']:
print "failed to add replica: %s" % res['Message']
return
for lfn in sorted( res['Value']['Failed'].keys() ):
error = res['Value']['Failed'][lfn]
print "failed to add replica: %s" % ( error )
for lfn in sorted( res['Value']['Successful'].keys() ):
print "added %s" % lfn
def do_removeReplica( self, args ):
"""Remove replica from the transformation DB
usage: removeReplica <lfn> <se>
"""
argss = args.split()
if not len( argss ) == 2:
print "no file info supplied"
return
lfn = argss[0]
se = argss[1]
lfnDict = {}
lfnDict[lfn] = {'PFN':'IGNORED-PFN', 'SE':se, 'Size':0, 'GUID':'IGNORED-GUID', 'Checksum':'IGNORED-CHECKSUM'}
res = self.server.removeReplica( lfnDict )
if not res['OK']:
print "failed to remove replica: %s" % res['Message']
return
for lfn in sorted( res['Value']['Failed'].keys() ):
error = res['Value']['Failed'][lfn]
print "failed to remove replica: %s" % ( error )
for lfn in sorted( res['Value']['Successful'].keys() ):
print "removed %s" % lfn
def do_setReplicaStatus( self, args ):
"""Set replica status, usually used to mark a replica Problematic
usage: setReplicaStatus <lfn> <status> <se>
"""
argss = args.split()
if not len( argss ) > 2:
print "no file info supplied"
return
lfn = argss[0]
status = argss[1]
se = argss[2]
lfnDict = {}
lfnDict[lfn] = {'Status':status, 'PFN':'IGNORED-PFN', 'SE':se, 'Size':0, 'GUID':'IGNORED-GUID', 'Checksum':'IGNORED-CHECKSUM'}
res = self.server.setReplicaStatus( lfnDict )
if not res['OK']:
print "failed to set replica status: %s" % res['Message']
return
for lfn in sorted( res['Value']['Failed'].keys() ):
error = res['Value']['Failed'][lfn]
print "failed to set replica status: %s" % ( error )
for lfn in sorted( res['Value']['Successful'].keys() ):
print "updated replica status %s" % lfn
class TransformationCLI(cmd.Cmd, API):
def __init__(self):
self.server = TransformationClient()
self.indentSpace = 4
cmd.Cmd.__init__(self)
def printPair(self, key, value, separator=":"):
valueList = value.split("\n")
print "%s%s%s %s" % (key, " " * (self.indentSpace - len(key)),
separator, valueList[0].strip())
for valueLine in valueList[1:-1]:
print "%s %s" % (" " * self.indentSpace, valueLine.strip())
def do_exit(self, args):
""" Exits the shell.
usage: exit
"""
sys.exit(0)
def do_quit(self, *args):
""" Exits the shell.
Usage: quit
"""
sys.exit(0)
def do_help(self, args):
""" Default version of the help command
Usage: help <command>
OR use helpall to see description for all commans"""
cmd.Cmd.do_help(self, args)
# overriting default help command
def do_helpall(self, args):
"""
Shows help information
Usage: helpall <command>
If no command is specified all commands are shown
"""
if len(args) == 0:
print "\nAvailable commands:\n"
attrList = dir(self)
attrList.sort()
for attribute in attrList:
if attribute.find("do_") == 0:
self.printPair(attribute[3:],
getattr(self, attribute).__doc__[1:])
print ""
else:
command = args.split()[0].strip()
try:
obj = getattr(self, "do_%s" % command)
except:
print "There's no such %s command" % command
return
self.printPair(command, obj.__doc__[1:])
def do_shell(self, args):
"""Execute a shell command
usage !<shell_command>
"""
comm = args
res = shellCall(0, comm)
if res['OK'] and res['Value'][0] == 0:
returnCode, stdOut, stdErr = res['Value']
print "%s\n%s" % (stdOut, stdErr)
else:
print res['Message']
def check_params(self, args, num):
"""Checks if the number of parameters correct"""
argss = string.split(args)
length = len(argss)
if length < num:
print "Error: Number of arguments provided %d less that required %d, please correct." % (
length, num)
return (False, length)
return (argss, length)
def check_id_or_name(self, id_or_name):
"""resolve name or Id by converting type of argument """
if id_or_name.isdigit():
return long(id_or_name) # its look like id
return id_or_name
def do_setServer(self, args):
""" Set the destination server
usage: setServer serverURL
"""
argss = string.split(args)
if len(argss) == 0:
print "no server provided"
self.serverURL = argss[0]
self.server.setServer(self.serverURL)
####################################################################
#
# These are the methods for transformation manipulation
#
def do_getall(self, args):
"""Get transformation details
usage: getall [Status] [Status]
"""
oTrans = Transformation()
oTrans.setServer(self.serverURL)
oTrans.getTransformations(transStatus=string.split(args),
printOutput=True)
def do_getStatus(self, args):
"""Get transformation details
usage: getStatus <transName|ID>
"""
argss = string.split(args)
if not len(argss) > 0:
print "no transformation supplied"
return
for transName in argss:
res = self.server.getTransformation(transName)
if not res['OK']:
print "Getting status of %s failed: %s" % (transName,
res['Message'])
else:
print "%s: %s" % (transName, res['Value']['Status'])
def do_setStatus(self, args):
"""Set transformation status
usage: setStatus <Status> <transName|ID>
Status <'New' 'Active' 'Stopped' 'Completed' 'Cleaning'>
"""
argss = string.split(args)
if not len(argss) > 1:
print "transformation and status not supplied"
return
status = argss[0]
transNames = argss[1:]
for transName in transNames:
res = self.server.setTransformationParameter(
transName, 'Status', status)
if not res['OK']:
print "Setting status of %s failed: %s" % (transName,
res['Message'])
else:
print "%s set to %s" % (transName, status)
def do_start(self, args):
"""Start transformation
usage: start <transName|ID>
"""
argss = string.split(args)
if not len(argss) > 0:
print "no transformation supplied"
return
for transName in argss:
res = self.server.setTransformationParameter(
transName, 'Status', 'Active')
if not res['OK']:
print "Setting Status of %s failed: %s" % (transName,
res['Message'])
else:
res = self.server.setTransformationParameter(
transName, 'AgentType', 'Automatic')
if not res['OK']:
print "Setting AgentType of %s failed: %s" % (
transName, res['Message'])
else:
print "%s started" % transName
def do_stop(self, args):
"""Stop transformation
usage: stop <transID|ID>
"""
argss = string.split(args)
if not len(argss) > 0:
print "no transformation supplied"
return
if not len(argss) > 0:
print "no transformation supplied"
return
for transName in argss:
res = self.server.setTransformationParameter(
transName, 'AgentType', 'Manual')
if not res['OK']:
print "Stopping of %s failed: %s" % (transName, res['Message'])
else:
print "%s stopped" % transName
def do_flush(self, args):
"""Flush transformation
usage: flush <transName|ID>
"""
argss = string.split(args)
if not len(argss) > 0:
print "no transformation supplied"
return
for transName in argss:
res = self.server.setTransformationParameter(
transName, 'Status', 'Flush')
if not res['OK']:
print "Flushing of %s failed: %s" % (transName, res['Message'])
else:
print "%s flushing" % transName
def do_get(self, args):
"""Get transformation definition
usage: get <transName|ID>
"""
argss = string.split(args)
if not len(argss) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformation(transName)
if not res['OK']:
print "Failed to get %s: %s" % (transName, res['Message'])
else:
res['Value'].pop('Body')
printDict(res['Value'])
def do_getBody(self, args):
"""Get transformation body
usage: getBody <transName|ID>
"""
argss = string.split(args)
if not len(argss) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformation(transName)
if not res['OK']:
print "Failed to get %s: %s" % (transName, res['Message'])
else:
print res['Value']['Body']
def do_getFileStat(self, args):
"""Get transformation file statistics
usage: getFileStat <transName|ID>
"""
argss = string.split(args)
if not len(argss) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformationStats(transName)
if not res['OK']:
print "Failed to get statistics for %s: %s" % (transName,
res['Message'])
else:
res['Value'].pop('Total')
printDict(res['Value'])
def do_modMask(self, args):
"""Modify transformation input definition
usage: modInput <mask> <transName|ID>
"""
argss = string.split(args)
if not len(argss) > 0:
print "no transformation supplied"
return
mask = argss[0]
transNames = argss[1:]
for transName in transNames:
res = self.server.setTransformationParameter(
transName, "FileMask", mask)
if not res['OK']:
print "Failed to modify input file mask for %s: %s" % (
transName, res['Message'])
else:
print "Updated %s filemask" % transName
def do_getFiles(self, args):
"""Get files for the transformation (optionally with a given status)
usage: getFiles <transName|ID> [Status] [Status]
"""
argss = string.split(args)
if not len(argss) > 0:
print "no transformation supplied"
return
transName = argss[0]
status = argss[1:]
res = self.server.getTransformation(transName)
if not res['OK']:
print "Failed to get transformation information: %s" % res[
'Message']
else:
selectDict = {'TransformationID': res['Value']['TransformationID']}
if status:
selectDict['Status'] = status
res = self.server.getTransformationFiles(condDict=selectDict)
if not res['OK']:
print "Failed to get transformation files: %s" % res['Message']
elif res['Value']:
self._printFormattedDictList(
res['Value'],
['LFN', 'Status', 'ErrorCount', 'TargetSE', 'LastUpdate'],
'LFN', 'LFN')
else:
print "No files found"
def do_getFileStatus(self, args):
"""Get file(s) status for the given transformation
usage: getFileStatus <transName|ID> <lfn> [<lfn>...]
"""
argss = string.split(args)
if len(argss) < 2:
print "transformation and file not supplied"
return
transName = argss[0]
lfns = argss[1:]
res = self.server.getTransformation(transName)
if not res['OK']:
print "Failed to get transformation information: %s" % res[
'Message']
else:
selectDict = {'TransformationID': res['Value']['TransformationID']}
res = self.server.getTransformationFiles(condDict=selectDict)
if not res['OK']:
print "Failed to get transformation files: %s" % res['Message']
elif res['Value']:
filesList = []
for fileDict in res['Value']:
if fileDict['LFN'] in lfns:
filesList.append(fileDict)
if filesList:
self._printFormattedDictList(filesList, [
'LFN', 'Status', 'ErrorCount', 'TargetSE', 'LastUpdate'
], 'LFN', 'LFN')
else:
print "Could not find any LFN in", lfns, "for transformation", transName
else:
print "No files found"
def do_setFileStatus(self, args):
"""Set file status for the given transformation
usage: setFileStatus <transName|ID> <lfn> <status>
"""
argss = string.split(args)
if not len(argss) == 3:
print "transformation file and status not supplied"
return
transName = argss[0]
lfn = argss[1]
status = argss[2]
res = self.server.setFileStatusForTransformation(
transName, status, [lfn])
if not res['OK']:
print "Failed to update file status: %s" % res['Message']
else:
print "Updated file status to %s" % status
def do_resetFile(self, args):
"""Reset file status for the given transformation
usage: setFileStatus <transName|ID> <lfn>
"""
argss = string.split(args)
if not len(argss) > 1:
print "transformation and file(s) not supplied"
return
transName = argss[0]
lfns = argss[1:]
res = self.server.setFileStatusForTransformation(
transName, 'Unused', lfns)
if not res['OK']:
print "Failed to reset file status: %s" % res['Message']
else:
print "Updated file statuses to 'Unused' for %d file(s)" % len(
lfns)
####################################################################
#
# These are the methods for file manipulation
#
def do_addDirectory(self, args):
"""Add files from the given catalog directory
usage: addDirectory <directory> [directory]
"""
argss = string.split(args)
if not len(argss) > 0:
print "no directory supplied"
return
for directory in argss:
res = self.server.addDirectory(directory, force=True)
if not res['OK']:
print 'failed to add directory %s: %s' % (directory,
res['Message'])
else:
print 'added %s files for %s' % (res['Value'], directory)
def do_replicas(self, args):
""" Get replicas for <path>
usage: replicas <lfn> [lfn]
"""
argss = string.split(args)
if not len(argss) > 0:
print "no files supplied"
return
res = self.server.getReplicas(argss)
if not res['OK']:
print "failed to get any replica information: %s" % res['Message']
return
for lfn in sortList(res['Value']['Failed'].keys()):
error = res['Value']['Failed'][lfn]
print "failed to get replica information for %s: %s" % (lfn, error)
for lfn in sortList(res['Value']['Successful'].keys()):
ses = sortList(res['Value']['Successful'][lfn].keys())
outStr = "%s :" % lfn.ljust(100)
for se in ses:
outStr = "%s %s" % (outStr, se.ljust(15))
print outStr
def do_addFile(self, args):
"""Add new files to transformation DB
usage: addFile <lfn> [lfn]
"""
argss = string.split(args)
if not len(argss) > 0:
print "no files supplied"
return
lfnDict = {}
for lfn in argss:
lfnDict[lfn] = {
'PFN': 'IGNORED-PFN',
'SE': 'IGNORED-SE',
'Size': 0,
'GUID': 'IGNORED-GUID',
'Checksum': 'IGNORED-CHECKSUM'
}
res = self.server.addFile(lfnDict, force=True)
if not res['OK']:
print "failed to add any files: %s" % res['Message']
return
for lfn in sortList(res['Value']['Failed'].keys()):
error = res['Value']['Failed'][lfn]
print "failed to add %s: %s" % (lfn, error)
for lfn in sortList(res['Value']['Successful'].keys()):
print "added %s" % lfn
def do_removeFile(self, args):
"""Remove file from transformation DB
usage: removeFile <lfn> [lfn]
"""
argss = string.split(args)
if not len(argss) > 0:
print "no files supplied"
return
res = self.server.removeFile(argss)
if not res['OK']:
print "failed to remove any files: %s" % res['Message']
return
for lfn in sortList(res['Value']['Failed'].keys()):
error = res['Value']['Failed'][lfn]
print "failed to remove %s: %s" % (lfn, error)
for lfn in sortList(res['Value']['Successful'].keys()):
print "removed %s" % lfn
def do_addReplica(self, args):
""" Add new replica to the transformation DB
usage: addReplica <lfn> <se>
"""
argss = string.split(args)
if not len(argss) == 2:
print "no file info supplied"
return
lfn = argss[0]
se = argss[1]
lfnDict = {}
lfnDict[lfn] = {
'PFN': 'IGNORED-PFN',
'SE': se,
'Size': 0,
'GUID': 'IGNORED-GUID',
'Checksum': 'IGNORED-CHECKSUM'
}
res = self.server.addReplica(lfnDict, force=True)
if not res['OK']:
print "failed to add replica: %s" % res['Message']
return
for lfn in sortList(res['Value']['Failed'].keys()):
error = res['Value']['Failed'][lfn]
print "failed to add replica: %s" % (error)
for lfn in sortList(res['Value']['Successful'].keys()):
print "added %s" % lfn
def do_removeReplica(self, args):
"""Remove replica from the transformation DB
usage: removeReplica <lfn> <se>
"""
argss = string.split(args)
if not len(argss) == 2:
print "no file info supplied"
return
lfn = argss[0]
se = argss[1]
lfnDict = {}
lfnDict[lfn] = {
'PFN': 'IGNORED-PFN',
'SE': se,
'Size': 0,
'GUID': 'IGNORED-GUID',
'Checksum': 'IGNORED-CHECKSUM'
}
res = self.server.removeReplica(lfnDict)
if not res['OK']:
print "failed to remove replica: %s" % res['Message']
return
for lfn in sortList(res['Value']['Failed'].keys()):
error = res['Value']['Failed'][lfn]
print "failed to remove replica: %s" % (error)
for lfn in sortList(res['Value']['Successful'].keys()):
print "removed %s" % lfn
def do_setReplicaStatus(self, args):
"""Set replica status, usually used to mark a replica Problematic
usage: setReplicaStatus <lfn> <status> <se>
"""
argss = string.split(args)
if not len(argss) > 2:
print "no file info supplied"
return
lfn = argss[0]
status = argss[1]
se = argss[2]
lfnDict = {}
lfnDict[lfn] = {
'Status': status,
'PFN': 'IGNORED-PFN',
'SE': se,
'Size': 0,
'GUID': 'IGNORED-GUID',
'Checksum': 'IGNORED-CHECKSUM'
}
res = self.server.setReplicaStatus(lfnDict)
if not res['OK']:
print "failed to set replica status: %s" % res['Message']
return
for lfn in sortList(res['Value']['Failed'].keys()):
error = res['Value']['Failed'][lfn]
print "failed to set replica status: %s" % (error)
for lfn in sortList(res['Value']['Successful'].keys()):
print "updated replica status %s" % lfn
selectDict = {'TransformationID': res['Value']['TransformationID']}
if status:
selectDict['Status'] = status
res = tc.getTransformationFiles(condDict=selectDict)
if not res['OK']:
gLogger.error('Failed to get transformation files: %s' % res['Message'])
continue
if not res['Value']:
gLogger.debug('No file found for transformation %s' % t)
continue
lfns = [f['LFN'] for f in res['Value']]
gLogger.notice('Reset files for status: %s' % status)
res = tc.setFileStatusForTransformation(t, 'Unused', lfns)
if not res['OK']:
gLogger.error('Failed to reset file status: %s' % res['Message'])
continue
if 'Failed' in res['Value']:
gLogger.warn('Could not reset some files: ')
for lfn, reason in res['Value']['Failed'].items():
gLogger.warn('%s: %s' % (lfn, reason))
gLogger.notice('Updated file statuses to "Unused" for %d file(s)' % len(lfns))
result = tc.setTransformationParameter(t, 'Status', 'Flush')
if not result['OK']:
gLogger.error('Can not flush transformation: %s' % result['Message'])
continue
DIRAC.exit(2)
lfns = [d['LFN'] for d in res['Value']]
if not lfns:
print "No files found in transformation %s, status %s" % (
transID, status)
if not lfns:
print "No files to be set in transformation", transID
else:
resetFiles = 0
failed = {}
for lfnChunk in breakListIntoChunks(lfns, 10000):
force = 'MaxReset' in status or 'Processed' in status or lfnsExplicit
res = transClient.setFileStatusForTransformation(transID,
newStatus,
lfnChunk,
force=force)
if res['OK']:
resetFiles += len(res['Value'].get('Successful',
res['Value']))
for lfn, reason in res['Value'].get('Failed',
{}).iteritems():
if reason != 'File not found in the Transformation Database':
failed.setdefault(reason, []).append(lfn)
else:
print "Failed to set %d files to %s in transformation %s: %s" % \
(len(lfns), newStatus, transID, res['Message'])
print "%d files were set %s in transformation %s" % (
resetFiles, newStatus, transID)
if failed:
for reason in failed:
class FileReport:
''' A stateful object for reporting to TransformationDB
'''
def __init__( self, server = 'Transformation/TransformationManager' ):
self.client = TransformationClient()
self.client.setServer( server )
self.statusDict = {}
self.transformation = None
def setFileStatus( self, transformation, lfn, status, sendFlag = False ):
''' Set file status in the context of the given transformation '''
if not self.transformation:
self.transformation = transformation
self.statusDict[lfn] = status
if sendFlag:
return self.commit()
return S_OK()
def setCommonStatus( self, status ):
''' Set common status for all files in the internal cache '''
for lfn in self.statusDict.keys():
self.statusDict[lfn] = status
return S_OK()
def getFiles( self ):
''' Get the statuses of the files already accumulated in the FileReport object '''
return copy.deepcopy( self.statusDict )
def commit( self ):
''' Commit pending file status update records '''
if not self.statusDict:
return S_OK()
# create intermediate status dictionary
sDict = {}
for lfn, status in self.statusDict.items():
if not sDict.has_key( status ):
sDict[status] = []
sDict[status].append( lfn )
summaryDict = {}
failedResults = []
for status, lfns in sDict.items():
res = self.client.setFileStatusForTransformation( self.transformation, status, lfns )
if not res['OK']:
failedResults.append( res )
continue
for lfn, error in res['Value']['Failed'].items():
gLogger.error( "Failed to update file status", "%s %s" % ( lfn, error ) )
if res['Value']['Successful']:
summaryDict[status] = len( res['Value']['Successful'] )
for lfn in res['Value']['Successful']:
self.statusDict.pop( lfn )
if not self.statusDict:
return S_OK( summaryDict )
result = S_ERROR( "Failed to update all file statuses" )
result['FailedResults'] = failedResults
return result
def generateRequest( self ):
''' Commit the accumulated records and generate request eventually '''
result = self.commit()
request = None
if not result['OK']:
# Generate Request
request = RequestContainer()
if result.has_key( 'FailedResults' ):
for res in result['FailedResults']:
if res.has_key( 'rpcStub' ):
request.setDISETRequest( res['rpcStub'] )
return S_OK( request )
class DataRecoveryAgent( AgentModule ):
def __init__(self, *args, **kwargs):
AgentModule.__init__( self, *args, **kwargs )
self.name = 'DataRecoveryAgent'
self.log = gLogger
#############################################################################
def initialize(self):
"""Sets defaults
"""
self.enableFlag = '' #defined below
self.replicaManager = ReplicaManager()
self.prodDB = TransformationClient()
self.requestClient = RequestClient()
self.taskIDName = 'TaskID'
self.externalStatus = 'ExternalStatus'
self.externalID = 'ExternalID'
self.am_setOption('PollingTime',2*60*60) #no stalled jobs are considered so can be frequent
self.enableFlag = self.am_getOption('EnableFlag', False)
self.am_setModuleParam("shifterProxy", "ProductionManager")
self.ops = Operations()
return S_OK()
#############################################################################
def execute(self):
""" The main execution method.
"""
self.log.info('Enable flag is %s' % self.enableFlag)
self.removalOKFlag = True
transformationTypes = ['MCReconstruction', 'MCSimulation', 'MCReconstruction_Overlay', 'Merge']
transformationStatus = ['Active', 'Completing']
fileSelectionStatus = ['Assigned', 'MaxReset']
updateStatus = 'Unused'
wmsStatusList = ['Failed']
#only worry about files > 12hrs since last update
selectDelay = self.am_getOption("Delay", 2) #hours
transformationDict = {}
for transStatus in transformationStatus:
result = self.getEligibleTransformations(transStatus, transformationTypes)
if not result['OK']:
self.log.error(result)
return S_ERROR('Could not obtain eligible transformations for status "%s"' % (transStatus))
if not result['Value']:
self.log.info('No "%s" transformations of types %s to process.' % (transStatus, string.join(transformationTypes, ', ')))
continue
transformationDict.update(result['Value'])
self.log.info('Selected %s transformations of types %s' % (len(transformationDict.keys()), string.join(transformationTypes, ', ')))
self.log.verbose('The following transformations were selected out of %s:\n%s' % (string.join(transformationTypes, ', '), string.join(transformationDict.keys(), ', ')))
trans = []
#initially this was useful for restricting the considered list
#now we use the DataRecoveryAgent in setups where IDs are low
ignoreLessThan = self.ops.getValue("Transformations/IgnoreLessThan", '724')
if trans:
self.log.info('Skipping all transformations except %s' % (string.join(trans, ', ')))
for transformation, typeName in transformationDict.items():
if trans:
if not transformation in trans:
continue
if ignoreLessThan:
if int(ignoreLessThan) > int(transformation):
self.log.verbose('Ignoring transformation %s ( is less than specified limit %s )' % (transformation, ignoreLessThan))
continue
self.log.info('='*len('Looking at transformation %s type %s:' % (transformation, typeName)))
self.log.info('Looking at transformation %s:' % (transformation))
result = self.selectTransformationFiles(transformation, fileSelectionStatus)
if not result['OK']:
self.log.error(result)
self.log.error('Could not select files for transformation %s' % transformation)
continue
if not result['Value']:
self.log.info('No files in status %s selected for transformation %s' % (string.join(fileSelectionStatus, ', '), transformation))
continue
fileDict = result['Value']
result = self.obtainWMSJobIDs(transformation, fileDict, selectDelay, wmsStatusList)
if not result['OK']:
self.log.error(result)
self.log.error('Could not obtain WMS jobIDs for files of transformation %s' % (transformation))
continue
if not result['Value']:
self.log.info('No eligible WMS jobIDs found for %s files in list:\n%s ...' % (len(fileDict.keys()), fileDict.keys()[0]))
continue
jobFileDict = result['Value']
fileCount = 0
for lfnList in jobFileDict.values():
fileCount += len(lfnList)
if not fileCount:
self.log.info('No files were selected for transformation %s after examining WMS jobs.' % transformation)
continue
self.log.info('%s files are selected after examining related WMS jobs' % (fileCount))
result = self.checkOutstandingRequests(jobFileDict)
if not result['OK']:
self.log.error(result)
continue
if not result['Value']:
self.log.info('No WMS jobs without pending requests to process.')
continue
jobFileNoRequestsDict = result['Value']
fileCount = 0
for lfnList in jobFileNoRequestsDict.values():
fileCount += len(lfnList)
self.log.info('%s files are selected after removing any relating to jobs with pending requests' % (fileCount))
result = self.checkDescendents(transformation, fileDict, jobFileNoRequestsDict)
if not result['OK']:
self.log.error(result)
continue
jobsWithFilesOKToUpdate = result['Value']['filesToMarkUnused']
jobsWithFilesProcessed = result['Value']['filesprocessed']
self.log.info('====> Transformation %s total files that can be updated now: %s' % (transformation, len(jobsWithFilesOKToUpdate)))
filesToUpdateUnused = []
for fileList in jobsWithFilesOKToUpdate:
filesToUpdateUnused.append(fileList)
if len(filesToUpdateUnused):
result = self.updateFileStatus(transformation, filesToUpdateUnused, updateStatus)
if not result['OK']:
self.log.error('Recoverable files were not updated with result:\n%s' % (result['Message']))
continue
else:
self.log.info('There are no files with failed jobs to update for production %s in this cycle' % transformation)
filesToUpdateProcessed = []
for fileList in jobsWithFilesProcessed:
filesToUpdateProcessed.append(fileList)
if len(filesToUpdateProcessed):
result = self.updateFileStatus(transformation, filesToUpdateProcessed, 'Processed')
if not result['OK']:
self.log.error('Recoverable files were not updated with result:\n%s' % (result['Message']))
continue
else:
self.log.info('There are no files processed to update for production %s in this cycle' % transformation)
return S_OK()
#############################################################################
def getEligibleTransformations(self, status, typeList):
""" Select transformations of given status and type.
"""
res = self.prodDB.getTransformations(condDict = {'Status' : status, 'Type' : typeList})
self.log.debug(res)
if not res['OK']:
return res
transformations = {}
for prod in res['Value']:
prodID = prod['TransformationID']
transformations[str(prodID)] = prod['Type']
return S_OK(transformations)
#############################################################################
def selectTransformationFiles(self, transformation, statusList):
""" Select files, production jobIDs in specified file status for a given transformation.
"""
#Until a query for files with timestamp can be obtained must rely on the
#WMS job last update
res = self.prodDB.getTransformationFiles(condDict = {'TransformationID' : transformation, 'Status' : statusList})
self.log.debug(res)
if not res['OK']:
return res
resDict = {}
for fileDict in res['Value']:
if not fileDict.has_key('LFN') or not fileDict.has_key(self.taskIDName) or not fileDict.has_key('LastUpdate'):
self.log.info('LFN, %s and LastUpdate are mandatory, >=1 are missing for:\n%s' % (self.taskIDName, fileDict))
continue
lfn = fileDict['LFN']
jobID = fileDict[self.taskIDName]
resDict[lfn] = jobID
if resDict:
self.log.info('Selected %s files overall for transformation %s' % (len(resDict.keys()), transformation))
return S_OK(resDict)
#############################################################################
def obtainWMSJobIDs(self, transformation, fileDict, selectDelay, wmsStatusList):
""" Group files by the corresponding WMS jobIDs, check the corresponding
jobs have not been updated for the delay time. Can't get into any
mess because we start from files only in MaxReset / Assigned and check
corresponding jobs. Mixtures of files for jobs in MaxReset and Assigned
statuses only possibly include some files in Unused status (not Processed
for example) that will not be touched.
"""
prodJobIDs = uniqueElements(fileDict.values())
self.log.info('The following %s production jobIDs apply to the selected files:\n%s' % (len(prodJobIDs), prodJobIDs))
jobFileDict = {}
condDict = {'TransformationID' : transformation, self.taskIDName : prodJobIDs}
delta = datetime.timedelta( hours = selectDelay )
now = dateTime()
olderThan = now-delta
res = self.prodDB.getTransformationTasks(condDict = condDict, older = olderThan,
timeStamp = 'LastUpdateTime', inputVector = True)
self.log.debug(res)
if not res['OK']:
self.log.error('getTransformationTasks returned an error:\n%s')
return res
for jobDict in res['Value']:
missingKey = False
for key in [self.taskIDName, self.externalID, 'LastUpdateTime', self.externalStatus, 'InputVector']:
if not jobDict.has_key(key):
self.log.info('Missing key %s for job dictionary, the following is available:\n%s' % (key, jobDict))
missingKey = True
continue
if missingKey:
continue
job = jobDict[self.taskIDName]
wmsID = jobDict[self.externalID]
lastUpdate = jobDict['LastUpdateTime']
wmsStatus = jobDict[self.externalStatus]
jobInputData = jobDict['InputVector']
jobInputData = [lfn.replace('LFN:','') for lfn in jobInputData.split(';')]
if not int(wmsID):
self.log.info('Prod job %s status is %s (ID = %s) so will not recheck with WMS' %(job, wmsStatus, wmsID))
continue
self.log.info('Job %s, prod job %s last update %s, production management system status %s' % (wmsID, job, lastUpdate, wmsStatus))
#Exclude jobs not having appropriate WMS status - have to trust that production management status is correct
if not wmsStatus in wmsStatusList:
self.log.info('Job %s is in status %s, not %s so will be ignored' % (wmsID, wmsStatus, string.join(wmsStatusList, ', ')))
continue
finalJobData = []
#Must map unique files -> jobs in expected state
for lfn,prodID in fileDict.items():
if int(prodID) == int(job):
finalJobData.append(lfn)
self.log.info('Found %s files for job %s' % (len(finalJobData), job))
jobFileDict[wmsID] = finalJobData
return S_OK(jobFileDict)
#############################################################################
def checkOutstandingRequests(self, jobFileDict):
""" Before doing anything check that no outstanding requests are pending
for the set of WMS jobIDs.
"""
jobs = jobFileDict.keys()
result = self.requestClient.getRequestForJobs(jobs)
if not result['OK']:
return result
if not result['Value']:
self.log.info('None of the jobs have pending requests')
return S_OK(jobFileDict)
for jobID in result['Value'].keys():
del jobFileDict[str(jobID)]
self.log.info('Removing jobID %s from consideration until requests are completed' % (jobID))
return S_OK(jobFileDict)
############################################################################
def checkDescendents(self, transformation, filedict, jobFileDict):
""" look that all jobs produced, or not output
"""
res = self.prodDB.getTransformationParameters(transformation, ['Body'])
if not res['OK']:
self.log.error('Could not get Body from TransformationDB')
return res
body = res['Value']
workflow = fromXMLString(body)
workflow.resolveGlobalVars()
olist = []
jtype = workflow.findParameter('JobType')
if not jtype:
self.log.error('Type for transformation %d was not defined' % transformation)
return S_ERROR('Type for transformation %d was not defined' % transformation)
for step in workflow.step_instances:
param = step.findParameter('listoutput')
if not param:
continue
olist.extend(param.value)
expectedlfns = []
contactfailed = []
fileprocessed = []
files = []
tasks_to_be_checked = {}
for files in jobFileDict.values():
for f in files:
if f in filedict:
tasks_to_be_checked[f] = filedict[f] #get the tasks that need to be checked
for filep, task in tasks_to_be_checked.items():
commons = {}
commons['outputList'] = olist
commons['PRODUCTION_ID'] = transformation
commons['JOB_ID'] = task
commons['JobType'] = jtype
out = constructProductionLFNs(commons)
expectedlfns = out['Value']['ProductionOutputData']
res = self.replicaManager.getCatalogFileMetadata(expectedlfns)
if not res['OK']:
self.log.error('Getting metadata failed')
contactfailed.append(filep)
continue
if not filep in files:
files.append(filep)
success = res['Value']['Successful'].keys()
failed = res['Value']['Failed'].keys()
if len(success) and not len(failed):
fileprocessed.append(filep)
final_list_unused = files
for file_all in files:
if file_all in fileprocessed:
try:
final_list_unused.remove(filep)
except:
self.log.warn("Item not in list anymore")
result = {'filesprocessed' : fileprocessed, 'filesToMarkUnused' : final_list_unused}
return S_OK(result)
#############################################################################
def updateFileStatus(self, transformation, fileList, fileStatus):
""" Update file list to specified status.
"""
if not self.enableFlag:
self.log.info('Enable flag is False, would update %s files to "%s" status for %s' % (len(fileList), fileStatus, transformation))
return S_OK()
self.log.info('Updating %s files to "%s" status for %s' % (len(fileList), fileStatus, transformation))
result = self.prodDB.setFileStatusForTransformation(int(transformation), fileStatus, fileList, force = True)
self.log.debug(result)
if not result['OK']:
self.log.error(result)
return result
if result['Value']['Failed']:
self.log.error(result['Value']['Failed'])
return result
msg = result['Value']['Successful']
for lfn, message in msg.items():
self.log.info('%s => %s' % (lfn, message))
return S_OK()
