def __dataQuery(self, prodid):
callback = {}
tsClient = TransformationClient()
res = tsClient.getTransformationInputDataQuery(prodid)
gLogger.info("-= #######", res)
if not res['OK']:
callback = {"success": "false", "error": res["Message"]}
else:
result = res["Value"]
back = []
for i in sorted(result.keys()):
back.append([i, result[i]])
callback = {"success": "true", "result": back}
return callback
def __dataQuery(self, prodid):
callback = {}
tsClient = TransformationClient()
res = tsClient.getTransformationInputDataQuery(prodid)
gLogger.info("-= #######", res)
if not res['OK']:
callback = {"success": "false", "error": res["Message"]}
else:
result = res["Value"]
back = []
for i in sorted(result.keys()):
back.append([i, result[i]])
callback = {"success": "true", "result": back}
return callback
class InputDataAgent( AgentModule ):
def __init__( self, *args, **kwargs ):
''' c'tor
'''
AgentModule.__init__( self, *args, **kwargs )
self.fileLog = {}
self.timeLog = {}
self.fullTimeLog = {}
self.pollingTime = self.am_getOption( 'PollingTime', 120 )
self.fullUpdatePeriod = self.am_getOption( 'FullUpdatePeriod', 86400 )
self.refreshonly = self.am_getOption( 'RefreshOnly', False )
self.dateKey = self.am_getOption( 'DateKey', None )
self.transClient = TransformationClient()
self.metadataClient = FileCatalogClient()
self.transformationTypes = None
#############################################################################
def initialize( self ):
''' Make the necessary initializations
'''
gMonitor.registerActivity( "Iteration", "Agent Loops", AGENT_NAME, "Loops/min", gMonitor.OP_SUM )
agentTSTypes = self.am_getOption( 'TransformationTypes', [] )
if agentTSTypes:
self.transformationTypes = sorted( agentTSTypes )
else:
dataProc = Operations().getValue( 'Transformations/DataProcessing', ['MCSimulation', 'Merge'] )
dataManip = Operations().getValue( 'Transformations/DataManipulation', ['Replication', 'Removal'] )
self.transformationTypes = sorted( dataProc + dataManip )
extendables = Operations().getValue( 'Transformations/ExtendableTransfTypes', [])
if extendables:
for extendable in extendables:
if extendable in self.transformationTypes:
self.transformationTypes.remove(extendable)
#This is because the Extendables do not use this Agent (have no Input data query)
return S_OK()
##############################################################################
def execute( self ):
''' Main execution method
'''
gMonitor.addMark( 'Iteration', 1 )
# Get all the transformations
result = self.transClient.getTransformations( {'Status' : 'Active',
'Type' : self.transformationTypes } )
if not result['OK']:
gLogger.error( "InputDataAgent.execute: Failed to get transformations.", result['Message'] )
return S_OK()
# Process each transformation
for transDict in result['Value']:
transID = long( transDict['TransformationID'] )
res = self.transClient.getTransformationInputDataQuery( transID )
if not res['OK']:
if res['Message'] == 'No InputDataQuery found for transformation':
gLogger.info( "InputDataAgent.execute: No input data query found for transformation %d" % transID )
else:
gLogger.error( "InputDataAgent.execute: Failed to get input data query for %d" % transID, res['Message'] )
continue
inputDataQuery = res['Value']
if self.refreshonly:
# Determine the correct time stamp to use for this transformation
if self.timeLog.has_key( transID ):
if self.fullTimeLog.has_key( transID ):
# If it is more than a day since the last reduced query, make a full query just in case
if ( datetime.datetime.utcnow() - self.fullTimeLog[transID] ) < datetime.timedelta( seconds = self.fullUpdatePeriod ):
timeStamp = self.timeLog[transID]
if self.dateKey:
inputDataQuery[self.dateKey] = ( timeStamp - datetime.timedelta( seconds = 10 ) ).strftime( '%Y-%m-%d %H:%M:%S' )
else:
gLogger.error( "DateKey was not set in the CS, cannot use the RefreshOnly" )
else:
self.fullTimeLog[transID] = datetime.datetime.utcnow()
self.timeLog[transID] = datetime.datetime.utcnow()
if not self.fullTimeLog.has_key( transID ):
self.fullTimeLog[transID] = datetime.datetime.utcnow()
# Perform the query to the metadata catalog
gLogger.verbose( "Using input data query for transformation %d: %s" % ( transID, str( inputDataQuery ) ) )
start = time.time()
result = self.metadataClient.findFilesByMetadata( inputDataQuery )
rtime = time.time() - start
gLogger.verbose( "Metadata catalog query time: %.2f seconds." % ( rtime ) )
if not result['OK']:
gLogger.error( "InputDataAgent.execute: Failed to get response from the metadata catalog", result['Message'] )
continue
lfnList = result['Value']
# Check if the number of files has changed since the last cycle
nlfns = len( lfnList )
gLogger.info( "%d files returned for transformation %d from the metadata catalog" % ( nlfns, int( transID ) ) )
if self.fileLog.has_key( transID ):
if nlfns == self.fileLog[transID]:
gLogger.verbose( 'No new files in metadata catalog since last check' )
self.fileLog[transID] = nlfns
# Add any new files to the transformation
addedLfns = []
if lfnList:
gLogger.verbose( 'Processing %d lfns for transformation %d' % ( len( lfnList ), transID ) )
# Add the files to the transformation
gLogger.verbose( 'Adding %d lfns for transformation %d' % ( len( lfnList ), transID ) )
result = self.transClient.addFilesToTransformation( transID, sorted( lfnList ) )
if not result['OK']:
gLogger.warn( "InputDataAgent.execute: failed to add lfns to transformation", result['Message'] )
self.fileLog[transID] = 0
else:
if result['Value']['Failed']:
for lfn, error in res['Value']['Failed'].items():
gLogger.warn( "InputDataAgent.execute: Failed to add %s to transformation" % lfn, error )
if result['Value']['Successful']:
for lfn, status in result['Value']['Successful'].items():
if status == 'Added':
addedLfns.append( lfn )
gLogger.info( "InputDataAgent.execute: Added %d files to transformation" % len( addedLfns ) )
return S_OK()
class TransformationCLI(CLI, API):
def __init__(self):
self.server = TransformationClient()
self.indentSpace = 4
CLI.__init__(self)
API.__init__(self)
def printPair(self, key, value, separator=":"):
valueList = value.split("\n")
print "%s%s%s %s" % (key, " " * (self.indentSpace - len(key)),
separator, valueList[0].strip())
for valueLine in valueList[1:-1]:
print "%s %s" % (" " * self.indentSpace, valueLine.strip())
def do_help(self, args):
""" Default version of the help command
Usage: help <command>
OR use helpall to see description for all commands"""
CLI.do_help(self, args)
# overriting default help command
def do_helpall(self, args):
"""
Shows help information
Usage: helpall <command>
If no command is specified all commands are shown
"""
if len(args) == 0:
print "\nAvailable commands:\n"
attrList = dir(self)
attrList.sort()
for attribute in attrList:
if attribute.find("do_") == 0:
self.printPair(attribute[3:],
getattr(self, attribute).__doc__[1:])
print ""
else:
command = args.split()[0].strip()
try:
obj = getattr(self, "do_%s" % command)
except:
print "There's no such %s command" % command
return
self.printPair(command, obj.__doc__[1:])
def do_shell(self, args):
"""Execute a shell command
usage !<shell_command>
"""
comm = args
res = shellCall(0, comm)
if res['OK'] and res['Value'][0] == 0:
_returnCode, stdOut, stdErr = res['Value']
print "%s\n%s" % (stdOut, stdErr)
else:
print res['Message']
def check_params(self, args, num):
"""Checks if the number of parameters correct"""
argss = args.split()
length = len(argss)
if length < num:
print "Error: Number of arguments provided %d less that required %d, please correct." % (
length, num)
return (False, length)
return (argss, length)
def check_id_or_name(self, id_or_name):
"""resolve name or Id by converting type of argument """
if id_or_name.isdigit():
return long(id_or_name) # its look like id
return id_or_name
####################################################################
#
# These are the methods for transformation manipulation
#
def do_getall(self, args):
"""Get transformation details
usage: getall [Status] [Status]
"""
oTrans = Transformation()
oTrans.getTransformations(transStatus=args.split(), printOutput=True)
def do_getAllByUser(self, args):
"""Get all transformations created by a given user
The first argument is the authorDN or username. The authorDN
is preferred: it need to be inside quotes because contains
white spaces. Only authorDN should be quoted.
When the username is provided instead,
the authorDN is retrieved from the uploaded proxy,
so that the retrieved transformations are those created by
the user who uploaded that proxy: that user could be different
that the username provided to the function.
usage: getAllByUser authorDN or username [Status] [Status]
"""
oTrans = Transformation()
argss = args.split()
username = ""
author = ""
status = []
if not len(argss) > 0:
print self.do_getAllByUser.__doc__
return
# if the user didnt quoted the authorDN ends
if '=' in argss[0] and argss[0][0] not in ["'", '"']:
print "AuthorDN need to be quoted (just quote that argument)"
return
if argss[0][0] in ["'", '"']: # authorDN given
author = argss[0]
status_idx = 1
for arg in argss[1:]:
author += ' ' + arg
status_idx += 1
if arg[-1] in ["'", '"']:
break
# At this point we should have something like 'author'
if not author[0] in ["'", '"'] or not author[-1] in ["'", '"']:
print "AuthorDN need to be quoted (just quote that argument)"
return
else:
author = author[1:-1] # throw away the quotes
# the rest are the requested status
status = argss[status_idx:]
else: # username given
username = argss[0]
status = argss[1:]
oTrans.getTransformationsByUser(authorDN=author,
userName=username,
transStatus=status,
printOutput=True)
def do_summaryTransformations(self, args):
"""Show the summary for a list of Transformations
Fields starting with 'F' ('J') refers to files (jobs).
Proc. stand for processed.
Usage: summaryTransformations <ProdID> [<ProdID> ...]
"""
argss = args.split()
if not len(argss) > 0:
print self.do_summaryTransformations.__doc__
return
transid = argss
oTrans = Transformation()
oTrans.getSummaryTransformations(transID=transid)
def do_getStatus(self, args):
"""Get transformation details
usage: getStatus <transName|ID>
"""
argss = args.split()
if not len(argss) > 0:
print "no transformation supplied"
return
for transName in argss:
res = self.server.getTransformation(transName)
if not res['OK']:
print "Getting status of %s failed: %s" % (transName,
res['Message'])
else:
print "%s: %s" % (transName, res['Value']['Status'])
def do_setStatus(self, args):
"""Set transformation status
usage: setStatus <Status> <transName|ID>
Status <'New' 'Active' 'Stopped' 'Completed' 'Cleaning'>
"""
argss = args.split()
if not len(argss) > 1:
print "transformation and status not supplied"
return
status = argss[0]
transNames = argss[1:]
for transName in transNames:
res = self.server.setTransformationParameter(
transName, 'Status', status)
if not res['OK']:
print "Setting status of %s failed: %s" % (transName,
res['Message'])
else:
print "%s set to %s" % (transName, status)
def do_start(self, args):
"""Start transformation
usage: start <transName|ID>
"""
argss = args.split()
if not len(argss) > 0:
print "no transformation supplied"
return
for transName in argss:
res = self.server.setTransformationParameter(
transName, 'Status', 'Active')
if not res['OK']:
print "Setting Status of %s failed: %s" % (transName,
res['Message'])
else:
res = self.server.setTransformationParameter(
transName, 'AgentType', 'Automatic')
if not res['OK']:
print "Setting AgentType of %s failed: %s" % (
transName, res['Message'])
else:
print "%s started" % transName
def do_stop(self, args):
"""Stop transformation
usage: stop <transID|ID>
"""
argss = args.split()
if not len(argss) > 0:
print "no transformation supplied"
return
if not len(argss) > 0:
print "no transformation supplied"
return
for transName in argss:
res = self.server.setTransformationParameter(
transName, 'AgentType', 'Manual')
if not res['OK']:
print "Stopping of %s failed: %s" % (transName, res['Message'])
else:
print "%s stopped" % transName
def do_flush(self, args):
"""Flush transformation
usage: flush <transName|ID>
"""
argss = args.split()
if not len(argss) > 0:
print "no transformation supplied"
return
for transName in argss:
res = self.server.setTransformationParameter(
transName, 'Status', 'Flush')
if not res['OK']:
print "Flushing of %s failed: %s" % (transName, res['Message'])
else:
print "%s flushing" % transName
def do_get(self, args):
"""Get transformation definition
usage: get <transName|ID>
"""
argss = args.split()
if not len(argss) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformation(transName)
if not res['OK']:
print "Failed to get %s: %s" % (transName, res['Message'])
else:
res['Value'].pop('Body')
printDict(res['Value'])
def do_getBody(self, args):
"""Get transformation body
usage: getBody <transName|ID>
"""
argss = args.split()
if not len(argss) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformation(transName)
if not res['OK']:
print "Failed to get %s: %s" % (transName, res['Message'])
else:
print res['Value']['Body']
def do_getFileStat(self, args):
"""Get transformation file statistics
usage: getFileStat <transName|ID>
"""
argss = args.split()
if not len(argss) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformationStats(transName)
if not res['OK']:
print "Failed to get statistics for %s: %s" % (transName,
res['Message'])
else:
res['Value'].pop('Total')
printDict(res['Value'])
def do_modMask(self, args):
"""Modify transformation input definition
usage: modInput <mask> <transName|ID>
"""
argss = args.split()
if not len(argss) > 0:
print "no transformation supplied"
return
mask = argss[0]
transNames = argss[1:]
for transName in transNames:
res = self.server.setTransformationParameter(
transName, "FileMask", mask)
if not res['OK']:
print "Failed to modify input file mask for %s: %s" % (
transName, res['Message'])
else:
print "Updated %s filemask" % transName
def do_getFiles(self, args):
"""Get files for the transformation (optionally with a given status)
usage: getFiles <transName|ID> [Status] [Status]
"""
argss = args.split()
if not len(argss) > 0:
print "no transformation supplied"
return
transName = argss[0]
status = argss[1:]
res = self.server.getTransformation(transName)
if not res['OK']:
print "Failed to get transformation information: %s" % res[
'Message']
else:
selectDict = {'TransformationID': res['Value']['TransformationID']}
if status:
selectDict['Status'] = status
res = self.server.getTransformationFiles(condDict=selectDict)
if not res['OK']:
print "Failed to get transformation files: %s" % res['Message']
elif res['Value']:
self._printFormattedDictList(
res['Value'],
['LFN', 'Status', 'ErrorCount', 'TargetSE', 'LastUpdate'],
'LFN', 'LFN')
else:
print "No files found"
def do_getFileStatus(self, args):
"""Get file(s) status for the given transformation
usage: getFileStatus <transName|ID> <lfn> [<lfn>...]
"""
argss = args.split()
if len(argss) < 2:
print "transformation and file not supplied"
return
transName = argss[0]
lfns = argss[1:]
res = self.server.getTransformation(transName)
if not res['OK']:
print "Failed to get transformation information: %s" % res[
'Message']
else:
selectDict = {'TransformationID': res['Value']['TransformationID']}
res = self.server.getTransformationFiles(condDict=selectDict)
if not res['OK']:
print "Failed to get transformation files: %s" % res['Message']
elif res['Value']:
filesList = []
for fileDict in res['Value']:
if fileDict['LFN'] in lfns:
filesList.append(fileDict)
if filesList:
self._printFormattedDictList(filesList, [
'LFN', 'Status', 'ErrorCount', 'TargetSE', 'LastUpdate'
], 'LFN', 'LFN')
else:
print "Could not find any LFN in", lfns, "for transformation", transName
else:
print "No files found"
def do_getOutputFiles(self, args):
"""Get output files for the transformation
usage: getOutputFiles <transName|ID>
"""
argss = args.split()
if not len(argss) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformation(transName)
if not res['OK']:
print "Failed to get transformation information: %s" % res[
'Message']
else:
fc = FileCatalog()
meta = {}
meta['ProdID'] = transName
res = fc.findFilesByMetadata(meta)
if not res['OK']:
print res['Message']
return
if not len(res['Value']) > 0:
print 'No output files yet for transformation %d' % int(
transName)
return
else:
for lfn in res['Value']:
print lfn
def do_getInputDataQuery(self, args):
"""Get input data query for the transformation
usage: getInputDataQuery <transName|ID>
"""
argss = args.split()
if not len(argss) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformationInputDataQuery(transName)
if not res['OK']:
print "Failed to get transformation input data query: %s" % res[
'Message']
else:
print res['Value']
def do_setFileStatus(self, args):
"""Set file status for the given transformation
usage: setFileStatus <transName|ID> <lfn> <status>
"""
argss = args.split()
if not len(argss) == 3:
print "transformation file and status not supplied"
return
transName = argss[0]
lfn = argss[1]
status = argss[2]
res = self.server.setFileStatusForTransformation(
transName, status, [lfn])
if not res['OK']:
print "Failed to update file status: %s" % res['Message']
else:
print "Updated file status to %s" % status
def do_resetFile(self, args):
"""Reset file status for the given transformation
usage: resetFile <transName|ID> <lfns>
"""
argss = args.split()
if not len(argss) > 1:
print "transformation and file(s) not supplied"
return
transName = argss[0]
lfns = argss[1:]
res = self.server.setFileStatusForTransformation(
transName, 'Unused', lfns)
if not res['OK']:
print "Failed to reset file status: %s" % res['Message']
else:
if 'Failed' in res['Value']:
print "Could not reset some files: "
for lfn, reason in res['Value']['Failed'].items():
print lfn, reason
else:
print "Updated file statuses to 'Unused' for %d file(s)" % len(
lfns)
def do_resetProcessedFile(self, args):
""" Reset file status for the given transformation
usage: resetFile <transName|ID> <lfn>
"""
argss = args.split()
if not len(argss) > 1:
print "transformation and file(s) not supplied"
return
transName = argss[0]
lfns = argss[1:]
res = self.server.setFileStatusForTransformation(transName,
'Unused',
lfns,
force=True)
if not res['OK']:
print "Failed to reset file status: %s" % res['Message']
else:
if 'Failed' in res['Value'] and res['Value']['Failed']:
print "Could not reset some files: "
for lfn, reason in res['Value']['Failed'].items():
print lfn, reason
else:
print "Updated file statuses to 'Unused' for %d file(s)" % len(
lfns)
####################################################################
#
# These are the methods for file manipulation
#
def do_addDirectory(self, args):
"""Add files from the given catalog directory
usage: addDirectory <directory> [directory]
"""
argss = args.split()
if not len(argss) > 0:
print "no directory supplied"
return
for directory in argss:
res = self.server.addDirectory(directory, force=True)
if not res['OK']:
print 'failed to add directory %s: %s' % (directory,
res['Message'])
else:
print 'added %s files for %s' % (res['Value'], directory)
def do_replicas(self, args):
""" Get replicas for <path>
usage: replicas <lfn> [lfn]
"""
argss = args.split()
if not len(argss) > 0:
print "no files supplied"
return
res = self.server.getReplicas(argss)
if not res['OK']:
print "failed to get any replica information: %s" % res['Message']
return
for lfn in sorted(res['Value']['Failed'].keys()):
error = res['Value']['Failed'][lfn]
print "failed to get replica information for %s: %s" % (lfn, error)
for lfn in sorted(res['Value']['Successful'].keys()):
ses = sorted(res['Value']['Successful'][lfn].keys())
outStr = "%s :" % lfn.ljust(100)
for se in ses:
outStr = "%s %s" % (outStr, se.ljust(15))
print outStr
def do_addFile(self, args):
"""Add new files to transformation DB
usage: addFile <lfn> [lfn]
"""
argss = args.split()
if not len(argss) > 0:
print "no files supplied"
return
lfnDict = {}
for lfn in argss:
lfnDict[lfn] = {
'PFN': 'IGNORED-PFN',
'SE': 'IGNORED-SE',
'Size': 0,
'GUID': 'IGNORED-GUID',
'Checksum': 'IGNORED-CHECKSUM'
}
res = self.server.addFile(lfnDict, force=True)
if not res['OK']:
print "failed to add any files: %s" % res['Message']
return
for lfn in sorted(res['Value']['Failed'].keys()):
error = res['Value']['Failed'][lfn]
print "failed to add %s: %s" % (lfn, error)
for lfn in sorted(res['Value']['Successful'].keys()):
print "added %s" % lfn
def do_removeFile(self, args):
"""Remove file from transformation DB
usage: removeFile <lfn> [lfn]
"""
argss = args.split()
if not len(argss) > 0:
print "no files supplied"
return
res = self.server.removeFile(argss)
if not res['OK']:
print "failed to remove any files: %s" % res['Message']
return
for lfn in sorted(res['Value']['Failed'].keys()):
error = res['Value']['Failed'][lfn]
print "failed to remove %s: %s" % (lfn, error)
for lfn in sorted(res['Value']['Successful'].keys()):
print "removed %s" % lfn
def do_addReplica(self, args):
""" Add new replica to the transformation DB
usage: addReplica <lfn> <se>
"""
argss = args.split()
if not len(argss) == 2:
print "no file info supplied"
return
lfn = argss[0]
se = argss[1]
lfnDict = {}
lfnDict[lfn] = {
'PFN': 'IGNORED-PFN',
'SE': se,
'Size': 0,
'GUID': 'IGNORED-GUID',
'Checksum': 'IGNORED-CHECKSUM'
}
res = self.server.addReplica(lfnDict, force=True)
if not res['OK']:
print "failed to add replica: %s" % res['Message']
return
for lfn in sorted(res['Value']['Failed'].keys()):
error = res['Value']['Failed'][lfn]
print "failed to add replica: %s" % (error)
for lfn in sorted(res['Value']['Successful'].keys()):
print "added %s" % lfn
def do_removeReplica(self, args):
"""Remove replica from the transformation DB
usage: removeReplica <lfn> <se>
"""
argss = args.split()
if not len(argss) == 2:
print "no file info supplied"
return
lfn = argss[0]
se = argss[1]
lfnDict = {}
lfnDict[lfn] = {
'PFN': 'IGNORED-PFN',
'SE': se,
'Size': 0,
'GUID': 'IGNORED-GUID',
'Checksum': 'IGNORED-CHECKSUM'
}
res = self.server.removeReplica(lfnDict)
if not res['OK']:
print "failed to remove replica: %s" % res['Message']
return
for lfn in sorted(res['Value']['Failed'].keys()):
error = res['Value']['Failed'][lfn]
print "failed to remove replica: %s" % (error)
for lfn in sorted(res['Value']['Successful'].keys()):
print "removed %s" % lfn
def do_setReplicaStatus(self, args):
"""Set replica status, usually used to mark a replica Problematic
usage: setReplicaStatus <lfn> <status> <se>
"""
argss = args.split()
if not len(argss) > 2:
print "no file info supplied"
return
lfn = argss[0]
status = argss[1]
se = argss[2]
lfnDict = {}
lfnDict[lfn] = {
'Status': status,
'PFN': 'IGNORED-PFN',
'SE': se,
'Size': 0,
'GUID': 'IGNORED-GUID',
'Checksum': 'IGNORED-CHECKSUM'
}
res = self.server.setReplicaStatus(lfnDict)
if not res['OK']:
print "failed to set replica status: %s" % res['Message']
return
for lfn in sorted(res['Value']['Failed'].keys()):
error = res['Value']['Failed'][lfn]
print "failed to set replica status: %s" % (error)
for lfn in sorted(res['Value']['Successful'].keys()):
print "updated replica status %s" % lfn
def _getProductionSummary():
clip = _Params()
clip.registerSwitch()
Script.parseCommandLine()
from ILCDIRAC.Core.Utilities.HTML import Table
from ILCDIRAC.Core.Utilities.ProcessList import ProcessList
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
from DIRAC import gConfig, gLogger
prod = clip.prod
full_detail = clip.full_det
fc = FileCatalogClient()
processlist = gConfig.getValue('/LocalSite/ProcessListPath')
prl = ProcessList(processlist)
processesdict = prl.getProcessesDict()
trc = TransformationClient()
prodids = []
if not prod:
conddict = {}
conddict['Status'] = clip.statuses
if clip.ptypes:
conddict['Type'] = clip.ptypes
res = trc.getTransformations(conddict)
if res['OK']:
for transfs in res['Value']:
prodids.append(transfs['TransformationID'])
else:
prodids.extend(prod)
metadata = []
gLogger.info("Will run on prods %s" % str(prodids))
for prodID in prodids:
if prodID < clip.minprod:
continue
meta = {}
meta['ProdID'] = prodID
res = trc.getTransformation(str(prodID))
if not res['OK']:
gLogger.error("Error getting transformation %s" % prodID)
continue
prodtype = res['Value']['Type']
proddetail = res['Value']['Description']
if prodtype == 'MCReconstruction' or prodtype == 'MCReconstruction_Overlay':
meta['Datatype'] = 'DST'
elif prodtype == 'MCGeneration':
meta['Datatype'] = 'gen'
elif prodtype == 'MCSimulation':
meta['Datatype'] = 'SIM'
elif prodtype in ['Split', 'Merge']:
gLogger.warn("Invalid query for %s productions" % prodtype)
continue
else:
gLogger.error("Unknown production type %s" % prodtype)
continue
res = fc.findFilesByMetadata(meta)
if not res['OK']:
gLogger.error(res['Message'])
continue
lfns = res['Value']
nb_files = len(lfns)
path = ""
if not len(lfns):
gLogger.warn("No files found for prod %s" % prodID)
continue
path = os.path.dirname(lfns[0])
res = fc.getDirectoryUserMetadata(path)
if not res['OK']:
gLogger.warn('No meta data found for %s' % path)
continue
dirmeta = {}
dirmeta['proddetail'] = proddetail
dirmeta['prodtype'] = prodtype
dirmeta['nb_files'] = nb_files
dirmeta.update(res['Value'])
lumi = 0.
nbevts = 0
addinfo = None
files = 0
xsec = 0.0
if not full_detail:
lfn = lfns[0]
info = _getFileInfo(lfn)
nbevts = info[1] * len(lfns)
lumi = info[0] * len(lfns)
addinfo = info[2]
if 'xsection' in addinfo:
if 'sum' in addinfo['xsection']:
if 'xsection' in addinfo['xsection']['sum']:
xsec += addinfo['xsection']['sum']['xsection']
files += 1
else:
for lfn in lfns:
info = _getFileInfo(lfn)
lumi += info[0]
nbevts += info[1]
addinfo = info[2]
if 'xsection' in addinfo:
if 'sum' in addinfo['xsection']:
if 'xsection' in addinfo['xsection']['sum']:
xsec += addinfo['xsection']['sum']['xsection']
files += 1
if not lumi:
xsec = 0
files = 0
depthDict = {}
depSet = set()
res = fc.getFileAncestors(lfns, [1, 2, 3, 4])
temp_ancestorlist = []
if res['OK']:
for lfn, ancestorsDict in res['Value']['Successful'].items():
for ancestor, dep in ancestorsDict.items():
depthDict.setdefault(dep, [])
if ancestor not in temp_ancestorlist:
depthDict[dep].append(ancestor)
depSet.add(dep)
temp_ancestorlist.append(ancestor)
depList = list(depSet)
depList.sort()
for ancestor in depthDict[depList[-1]]:
info = _getFileInfo(ancestor)
lumi += info[0]
addinfo = info[2]
if 'xsection' in addinfo:
if 'sum' in addinfo['xsection']:
if 'xsection' in addinfo['xsection']['sum']:
xsec += addinfo['xsection']['sum']['xsection']
files += 1
if xsec and files:
xsec /= files
dirmeta['CrossSection'] = xsec
else:
dirmeta['CrossSection'] = 0.0
if nbevts:
dirmeta['NumberOfEvents'] = nbevts
#if not lumi:
# dirmeta['Luminosity']=0
# dirmeta['CrossSection']=0
#else:
# if nbevts:
# dirmeta['CrossSection']=nbevts/lumi
# else:
# dirmeta['CrossSection']=0
#if addinfo:
# if 'xsection' in addinfo:
# if 'sum' in addinfo['xsection']:
# if 'xsection' in addinfo['xsection']['sum']:
# dirmeta['CrossSection']=addinfo['xsection']['sum']['xsection']
if 'NumberOfEvents' not in dirmeta:
dirmeta['NumberOfEvents'] = 0
#print processesdict[dirmeta['EvtType']]
dirmeta['detail'] = ''
if dirmeta['EvtType'] in processesdict:
if 'Detail' in processesdict[dirmeta['EvtType']]:
detail = processesdict[dirmeta['EvtType']]['Detail']
else:
detail = dirmeta['EvtType']
if not prodtype == 'MCGeneration':
res = trc.getTransformationInputDataQuery(str(prodID))
if res['OK']:
if 'ProdID' in res['Value']:
dirmeta['MomProdID'] = res['Value']['ProdID']
if 'MomProdID' not in dirmeta:
dirmeta['MomProdID'] = 0
dirmeta['detail'] = _translate(detail)
metadata.append(dirmeta)
detectors = {}
detectors['ILD'] = {}
corres = {
"MCGeneration": 'gen',
"MCSimulation": 'SIM',
"MCReconstruction": "REC",
"MCReconstruction_Overlay": "REC"
}
detectors['ILD']['SIM'] = []
detectors['ILD']['REC'] = []
detectors['SID'] = {}
detectors['SID']['SIM'] = []
detectors['SID']['REC'] = []
detectors['sid'] = {}
detectors['sid']['SIM'] = []
detectors['sid']['REC'] = []
detectors['gen'] = []
for channel in metadata:
if 'DetectorType' not in channel:
detectors['gen'].append(
(channel['detail'], channel['Energy'], channel['ProdID'],
channel['nb_files'],
channel['NumberOfEvents'] / channel['nb_files'],
channel['NumberOfEvents'], channel['CrossSection'],
str(channel['proddetail'])))
else:
if not channel['DetectorType'] in detectors:
gLogger.error("This is unknown detector",
channel['DetectorType'])
continue
detectors[channel['DetectorType']][corres[
channel['prodtype']]].append(
(channel['detail'], channel['Energy'],
channel['DetectorType'], channel['ProdID'],
channel['nb_files'],
channel['NumberOfEvents'] / channel['nb_files'],
channel['NumberOfEvents'], channel['CrossSection'],
channel['MomProdID'], str(channel['proddetail'])))
with open("tables.html", "w") as of:
of.write("""<!DOCTYPE html>
<html>
<head>
<title> Production summary </title>
</head>
<body>
""")
if len(detectors['gen']):
of.write("<h1>gen prods</h1>\n")
table = Table(header_row=('Channel', 'Energy', 'ProdID', 'Tasks',
'Average Evts/task', 'Statistics',
'Cross Section (fb)', 'Comment'))
for item in detectors['gen']:
table.rows.append(item)
of.write(str(table))
gLogger.info("Gen prods")
gLogger.info(str(table))
if len(detectors['ILD']):
of.write("<h1>ILD prods</h1>\n")
for ptype in detectors['ILD'].keys():
if len(detectors['ILD'][ptype]):
of.write("<h2>%s</h2>\n" % ptype)
table = Table(header_row=('Channel', 'Energy', 'Detector',
'ProdID', 'Number of Files',
'Events/File', 'Statistics',
'Cross Section (fb)',
'Origin ProdID', 'Comment'))
for item in detectors['ILD'][ptype]:
table.rows.append(item)
of.write(str(table))
gLogger.info("ILC CDR prods %s" % ptype)
gLogger.info(str(table))
if len(detectors['SID']):
of.write("<h1>SID prods</h1>\n")
for ptype in detectors['SID'].keys():
if len(detectors['SID'][ptype]):
of.write("<h2>%s</h2>\n" % ptype)
table = Table(header_row=('Channel', 'Energy', 'Detector',
'ProdID', 'Number of Files',
'Events/File', 'Statistics',
'Cross Section (fb)',
'Origin ProdID', 'Comment'))
for item in detectors['SID'][ptype]:
table.rows.append(item)
of.write(str(table))
gLogger.info("SID CDR prods %s" % ptype)
gLogger.info(str(table))
if len(detectors['sid']):
of.write("<h1>sid dbd prods</h1>\n")
for ptype in detectors['sid'].keys():
if len(detectors['sid'][ptype]):
of.write("<h2>%s</h2>\n" % ptype)
table = Table(header_row=('Channel', 'Energy', 'Detector',
'ProdID', 'Number of Files',
'Events/File', 'Statistics',
'Cross Section (fb)',
'Origin ProdID', 'Comment'))
for item in detectors['sid'][ptype]:
table.rows.append(item)
of.write(str(table))
gLogger.info("sid DBD prods %s" % ptype)
gLogger.info(str(table))
of.write("""
</body>
</html>
""")
gLogger.notice("Check ./tables.html in any browser for the results")
dexit(0)
# if 'xsection' in addinfo['xsection']['sum']:
# dirmeta['CrossSection']=addinfo['xsection']['sum']['xsection']
if not dirmeta.has_key('NumberOfEvents'):
dirmeta['NumberOfEvents']=0
#print processesdict[dirmeta['EvtType']]
dirmeta['detail']=''
if processesdict.has_key(dirmeta['EvtType']):
if processesdict[dirmeta['EvtType']].has_key('Detail'):
detail = processesdict[dirmeta['EvtType']]['Detail']
else:
detail=dirmeta['EvtType']
if not prodtype == 'MCGeneration':
res = trc.getTransformationInputDataQuery(str(prodID))
if res['OK']:
if res['Value'].has_key('ProdID'):
dirmeta['MomProdID']=res['Value']['ProdID']
if not dirmeta.has_key('MomProdID'):
dirmeta['MomProdID']=0
dirmeta['detail']= translate(detail)
metadata.append(dirmeta)
detectors = {}
detectors['ILD'] = {}
corres = {"MCGeneration":'gen',"MCSimulation":'SIM',"MCReconstruction":"REC","MCReconstruction_Overlay":"REC"}
detectors['ILD']['SIM'] = []
detectors['ILD']['REC'] = []
detectors['SID'] = {}
class InputDataAgent(AgentModule):
#############################################################################
def initialize(self):
""" Make the necessary initializations """
self.fileLog = {}
self.timeLog = {}
self.fullTimeLog = {}
self.pollingTime = self.am_getOption('PollingTime', 120)
self.fullUpdatePeriod = self.am_getOption('FullUpdatePeriod', 86400)
gMonitor.registerActivity("Iteration", "Agent Loops", AGENT_NAME,
"Loops/min", gMonitor.OP_SUM)
self.transClient = TransformationClient('TransformationDB')
self.metadataClient = FileCatalogClient()
return S_OK()
##############################################################################
def execute(self):
""" Main execution method
"""
gMonitor.addMark('Iteration', 1)
# Get all the transformations
result = self.transClient.getTransformations(
condDict={'Status': 'Active'})
activeTransforms = []
if not result['OK']:
gLogger.error(
"InputDataAgent.execute: Failed to get transformations.",
result['Message'])
return S_OK()
# Process each transformation
for transDict in result['Value']:
transID = long(transDict['TransformationID'])
res = self.transClient.getTransformationInputDataQuery(transID)
if not res['OK']:
if res['Message'] == 'No InputDataQuery found for transformation':
gLogger.info(
"InputDataAgent.execute: No input data query found for transformation %d"
% transID)
else:
gLogger.error(
"InputDataAgent.execute: Failed to get input data query for %d"
% transID, res['Message'])
continue
inputDataQuery = res['Value']
# Determine the correct time stamp to use for this transformation
if self.timeLog.has_key(transID):
if self.fullTimeLog.has_key(transID):
# If it is more than a day since the last reduced query, make a full query just in case
if (datetime.datetime.utcnow() -
self.fullTimeLog[transID]) < datetime.timedelta(
seconds=self.fullUpdatePeriod):
timeStamp = self.timeLog[transID]
inputDataQuery['StartDate'] = (
timeStamp - datetime.timedelta(seconds=10)
).strftime('%Y-%m-%d %H:%M:%S')
else:
self.fullTimeLog[transID] = datetime.datetime.utcnow()
self.timeLog[transID] = datetime.datetime.utcnow()
if not self.fullTimeLog.has_key(transID):
self.fullTimeLog[transID] = datetime.datetime.utcnow()
# Perform the query to the metadata catalog
gLogger.verbose(
"Using input data query for transformation %d: %s" %
(transID, str(inputDataQuery)))
start = time.time()
result = self.metadataClient.findFilesByMetadata(inputDataQuery)
rtime = time.time() - start
gLogger.verbose("Metadata catalog query time: %.2f seconds." %
(rtime))
if not result['OK']:
gLogger.error(
"InputDataAgent.execute: Failed to get response from the metadata catalog",
result['Message'])
continue
lfnList = result['Value']
# Check if the number of files has changed since the last cycle
nlfns = len(lfnList)
gLogger.info(
"%d files returned for transformation %d from the metadata catalog"
% (nlfns, int(transID)))
if self.fileLog.has_key(transID):
if nlfns == self.fileLog[transID]:
gLogger.verbose(
'No new files in metadata catalog since last check')
self.fileLog[transID] = nlfns
# Add any new files to the transformation
addedLfns = []
if lfnList:
gLogger.verbose('Processing %d lfns for transformation %d' %
(len(lfnList), transID))
# Add the files to the transformation
gLogger.verbose('Adding %d lfns for transformation %d' %
(len(lfnList), transID))
result = self.transClient.addFilesToTransformation(
transID, sortList(lfnList))
if not result['OK']:
gLogger.warn(
"InputDataAgent.execute: failed to add lfns to transformation",
result['Message'])
self.fileLog[transID] = 0
else:
if result['Value']['Failed']:
for lfn, error in res['Value']['Failed'].items():
gLogger.warn(
"InputDataAgent.execute: Failed to add %s to transformation"
% lfn, error)
if result['Value']['Successful']:
for lfn, status in result['Value']['Successful'].items(
):
if status == 'Added':
addedLfns.append(lfn)
gLogger.info(
"InputDataAgent.execute: Added %d files to transformation"
% len(addedLfns))
return S_OK()
class TransformationCLI( cmd.Cmd, API ):
def __init__( self ):
self.server = TransformationClient()
self.indentSpace = 4
cmd.Cmd.__init__( self )
API.__init__( self )
def printPair( self, key, value, separator = ":" ):
valueList = value.split( "\n" )
print "%s%s%s %s" % ( key, " " * ( self.indentSpace - len( key ) ), separator, valueList[0].strip() )
for valueLine in valueList[ 1:-1 ]:
print "%s %s" % ( " " * self.indentSpace, valueLine.strip() )
def do_exit( self, args ):
""" Exits the shell.
usage: exit
"""
sys.exit( 0 )
def do_quit( self, *args ):
""" Exits the shell.
Usage: quit
"""
sys.exit( 0 )
def do_help( self, args ):
""" Default version of the help command
Usage: help <command>
OR use helpall to see description for all commands"""
cmd.Cmd.do_help( self, args )
# overriting default help command
def do_helpall( self, args ):
"""
Shows help information
Usage: helpall <command>
If no command is specified all commands are shown
"""
if len( args ) == 0:
print "\nAvailable commands:\n"
attrList = dir( self )
attrList.sort()
for attribute in attrList:
if attribute.find( "do_" ) == 0:
self.printPair( attribute[ 3: ], getattr( self, attribute ).__doc__[ 1: ] )
print ""
else:
command = args.split()[0].strip()
try:
obj = getattr( self, "do_%s" % command )
except:
print "There's no such %s command" % command
return
self.printPair( command, obj.__doc__[1:] )
def do_shell( self, args ):
"""Execute a shell command
usage !<shell_command>
"""
comm = args
res = shellCall( 0, comm )
if res['OK'] and res['Value'][0] == 0:
_returnCode, stdOut, stdErr = res['Value']
print "%s\n%s" % ( stdOut, stdErr )
else:
print res['Message']
def check_params( self, args, num ):
"""Checks if the number of parameters correct"""
argss = args.split()
length = len( argss )
if length < num:
print "Error: Number of arguments provided %d less that required %d, please correct." % ( length, num )
return ( False, length )
return ( argss, length )
def check_id_or_name( self, id_or_name ):
"""resolve name or Id by converting type of argument """
if id_or_name.isdigit():
return long( id_or_name ) # its look like id
return id_or_name
####################################################################
#
# These are the methods for transformation manipulation
#
def do_getall( self, args ):
"""Get transformation details
usage: getall [Status] [Status]
"""
oTrans = Transformation()
oTrans.getTransformations( transStatus = args.split(), printOutput = True )
def do_getAllByUser( self, args ):
"""Get all transformations created by a given user
The first argument is the authorDN or username. The authorDN
is preferred: it need to be inside quotes because contains
white spaces. Only authorDN should be quoted.
When the username is provided instead,
the authorDN is retrieved from the uploaded proxy,
so that the retrieved transformations are those created by
the user who uploaded that proxy: that user could be different
that the username provided to the function.
usage: getAllByUser authorDN or username [Status] [Status]
"""
oTrans = Transformation()
argss = args.split()
username = ""
author = ""
status = []
if not len( argss ) > 0:
print self.do_getAllByUser.__doc__
return
# if the user didnt quoted the authorDN ends
if '=' in argss[0] and argss[0][0] not in ["'", '"']:
print "AuthorDN need to be quoted (just quote that argument)"
return
if argss[0][0] in ["'", '"']: # authorDN given
author = argss[0]
status_idx = 1
for arg in argss[1:]:
author += ' ' + arg
status_idx +=1
if arg[-1] in ["'", '"']:
break
# At this point we should have something like 'author'
if not author[0] in ["'", '"'] or not author[-1] in ["'", '"']:
print "AuthorDN need to be quoted (just quote that argument)"
return
else:
author = author[1:-1] # throw away the quotes
# the rest are the requested status
status = argss[ status_idx: ]
else: # username given
username = argss[0]
status = argss[ 1: ]
oTrans.getTransformationsByUser( authorDN = author, userName = username, transStatus = status, printOutput = True )
def do_summaryTransformations( self, args ):
"""Show the summary for a list of Transformations
Fields starting with 'F' ('J') refers to files (jobs).
Proc. stand for processed.
Usage: summaryTransformations <ProdID> [<ProdID> ...]
"""
argss = args.split()
if not len( argss ) > 0:
print self.do_summaryTransformations.__doc__
return
transid = argss
oTrans = Transformation()
oTrans.getSummaryTransformations( transID = transid )
def do_getStatus( self, args ):
"""Get transformation details
usage: getStatus <transName|ID>
"""
argss = args.split()
if not len( argss ) > 0:
print "no transformation supplied"
return
for transName in argss:
res = self.server.getTransformation( transName )
if not res['OK']:
print "Getting status of %s failed: %s" % ( transName, res['Message'] )
else:
print "%s: %s" % ( transName, res['Value']['Status'] )
def do_setStatus( self, args ):
"""Set transformation status
usage: setStatus <Status> <transName|ID>
Status <'New' 'Active' 'Stopped' 'Completed' 'Cleaning'>
"""
argss = args.split()
if not len( argss ) > 1:
print "transformation and status not supplied"
return
status = argss[0]
transNames = argss[1:]
for transName in transNames:
res = self.server.setTransformationParameter( transName, 'Status', status )
if not res['OK']:
print "Setting status of %s failed: %s" % ( transName, res['Message'] )
else:
print "%s set to %s" % ( transName, status )
def do_start( self, args ):
"""Start transformation
usage: start <transName|ID>
"""
argss = args.split()
if not len( argss ) > 0:
print "no transformation supplied"
return
for transName in argss:
res = self.server.setTransformationParameter( transName, 'Status', 'Active' )
if not res['OK']:
print "Setting Status of %s failed: %s" % ( transName, res['Message'] )
else:
res = self.server.setTransformationParameter( transName, 'AgentType', 'Automatic' )
if not res['OK']:
print "Setting AgentType of %s failed: %s" % ( transName, res['Message'] )
else:
print "%s started" % transName
def do_stop( self, args ):
"""Stop transformation
usage: stop <transID|ID>
"""
argss = args.split()
if not len( argss ) > 0:
print "no transformation supplied"
return
if not len( argss ) > 0:
print "no transformation supplied"
return
for transName in argss:
res = self.server.setTransformationParameter( transName, 'AgentType', 'Manual' )
if not res['OK']:
print "Stopping of %s failed: %s" % ( transName, res['Message'] )
else:
print "%s stopped" % transName
def do_flush( self, args ):
"""Flush transformation
usage: flush <transName|ID>
"""
argss = args.split()
if not len( argss ) > 0:
print "no transformation supplied"
return
for transName in argss:
res = self.server.setTransformationParameter( transName, 'Status', 'Flush' )
if not res['OK']:
print "Flushing of %s failed: %s" % ( transName, res['Message'] )
else:
print "%s flushing" % transName
def do_get( self, args ):
"""Get transformation definition
usage: get <transName|ID>
"""
argss = args.split()
if not len( argss ) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformation( transName )
if not res['OK']:
print "Failed to get %s: %s" % ( transName, res['Message'] )
else:
res['Value'].pop( 'Body' )
printDict( res['Value'] )
def do_getBody( self, args ):
"""Get transformation body
usage: getBody <transName|ID>
"""
argss = args.split()
if not len( argss ) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformation( transName )
if not res['OK']:
print "Failed to get %s: %s" % ( transName, res['Message'] )
else:
print res['Value']['Body']
def do_getFileStat( self, args ):
"""Get transformation file statistics
usage: getFileStat <transName|ID>
"""
argss = args.split()
if not len( argss ) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformationStats( transName )
if not res['OK']:
print "Failed to get statistics for %s: %s" % ( transName, res['Message'] )
else:
res['Value'].pop( 'Total' )
printDict( res['Value'] )
def do_modMask( self, args ):
"""Modify transformation input definition
usage: modInput <mask> <transName|ID>
"""
argss = args.split()
if not len( argss ) > 0:
print "no transformation supplied"
return
mask = argss[0]
transNames = argss[1:]
for transName in transNames:
res = self.server.setTransformationParameter( transName, "FileMask", mask )
if not res['OK']:
print "Failed to modify input file mask for %s: %s" % ( transName, res['Message'] )
else:
print "Updated %s filemask" % transName
def do_getFiles( self, args ):
"""Get files for the transformation (optionally with a given status)
usage: getFiles <transName|ID> [Status] [Status]
"""
argss = args.split()
if not len( argss ) > 0:
print "no transformation supplied"
return
transName = argss[0]
status = argss[1:]
res = self.server.getTransformation( transName )
if not res['OK']:
print "Failed to get transformation information: %s" % res['Message']
else:
selectDict = {'TransformationID':res['Value']['TransformationID']}
if status:
selectDict['Status'] = status
res = self.server.getTransformationFiles( condDict = selectDict )
if not res['OK']:
print "Failed to get transformation files: %s" % res['Message']
elif res['Value']:
self._printFormattedDictList( res['Value'], ['LFN', 'Status', 'ErrorCount', 'TargetSE', 'LastUpdate'],
'LFN', 'LFN' )
else:
print "No files found"
def do_getFileStatus( self, args ):
"""Get file(s) status for the given transformation
usage: getFileStatus <transName|ID> <lfn> [<lfn>...]
"""
argss = args.split()
if len( argss ) < 2:
print "transformation and file not supplied"
return
transName = argss[0]
lfns = argss[1:]
res = self.server.getTransformation( transName )
if not res['OK']:
print "Failed to get transformation information: %s" % res['Message']
else:
selectDict = {'TransformationID':res['Value']['TransformationID']}
res = self.server.getTransformationFiles( condDict = selectDict )
if not res['OK']:
print "Failed to get transformation files: %s" % res['Message']
elif res['Value']:
filesList = []
for fileDict in res['Value']:
if fileDict['LFN'] in lfns:
filesList.append( fileDict )
if filesList:
self._printFormattedDictList( filesList, ['LFN', 'Status', 'ErrorCount', 'TargetSE', 'LastUpdate'],
'LFN', 'LFN' )
else:
print "Could not find any LFN in", lfns, "for transformation", transName
else:
print "No files found"
def do_getOutputFiles( self, args ):
"""Get output files for the transformation
usage: getOutputFiles <transName|ID>
"""
argss = args.split()
if not len( argss ) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformation( transName )
if not res['OK']:
print "Failed to get transformation information: %s" % res['Message']
else:
fc = FileCatalog()
meta = {}
meta ['ProdID'] = transName
res = fc.findFilesByMetadata( meta )
if not res['OK']:
print res['Message']
return
if not len( res['Value'] ) > 0:
print 'No output files yet for transformation %d' %int(transName)
return
else:
for lfn in res['Value']:
print lfn
def do_getInputDataQuery( self, args ):
"""Get input data query for the transformation
usage: getInputDataQuery <transName|ID>
"""
argss = args.split()
if not len( argss ) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformationInputDataQuery( transName )
if not res['OK']:
print "Failed to get transformation input data query: %s" % res['Message']
else:
print res['Value']
def do_setFileStatus( self, args ):
"""Set file status for the given transformation
usage: setFileStatus <transName|ID> <lfn> <status>
"""
argss = args.split()
if not len( argss ) == 3:
print "transformation file and status not supplied"
return
transName = argss[0]
lfn = argss[1]
status = argss[2]
res = self.server.setFileStatusForTransformation( transName, status, [lfn] )
if not res['OK']:
print "Failed to update file status: %s" % res['Message']
else:
print "Updated file status to %s" % status
def do_resetFile( self, args ):
"""Reset file status for the given transformation
usage: resetFile <transName|ID> <lfns>
"""
argss = args.split()
if not len( argss ) > 1:
print "transformation and file(s) not supplied"
return
transName = argss[0]
lfns = argss[1:]
res = self.server.setFileStatusForTransformation( transName, 'Unused', lfns )
if not res['OK']:
print "Failed to reset file status: %s" % res['Message']
else:
if 'Failed' in res['Value']:
print "Could not reset some files: "
for lfn, reason in res['Value']['Failed'].items():
print lfn, reason
else:
print "Updated file statuses to 'Unused' for %d file(s)" % len( lfns )
def do_resetProcessedFile( self, args ):
""" Reset file status for the given transformation
usage: resetFile <transName|ID> <lfn>
"""
argss = args.split()
if not len( argss ) > 1:
print "transformation and file(s) not supplied"
return
transName = argss[0]
lfns = argss[1:]
res = self.server.setFileStatusForTransformation( transName, 'Unused', lfns, force = True )
if not res['OK']:
print "Failed to reset file status: %s" % res['Message']
else:
if res['Value']['Failed']:
print "Could not reset some files: "
for lfn, reason in res['Value']['Failed'].items():
print lfn, reason
else:
print "Updated file statuses to 'Unused' for %d file(s)" % len( lfns )
####################################################################
#
# These are the methods for file manipulation
#
def do_addDirectory( self, args ):
"""Add files from the given catalog directory
usage: addDirectory <directory> [directory]
"""
argss = args.split()
if not len( argss ) > 0:
print "no directory supplied"
return
for directory in argss:
res = self.server.addDirectory( directory, force = True )
if not res['OK']:
print 'failed to add directory %s: %s' % ( directory, res['Message'] )
else:
print 'added %s files for %s' % ( res['Value'], directory )
def do_replicas( self, args ):
""" Get replicas for <path>
usage: replicas <lfn> [lfn]
"""
argss = args.split()
if not len( argss ) > 0:
print "no files supplied"
return
res = self.server.getReplicas( argss )
if not res['OK']:
print "failed to get any replica information: %s" % res['Message']
return
for lfn in sorted( res['Value']['Failed'].keys() ):
error = res['Value']['Failed'][lfn]
print "failed to get replica information for %s: %s" % ( lfn, error )
for lfn in sorted( res['Value']['Successful'].keys() ):
ses = sorted( res['Value']['Successful'][lfn].keys() )
outStr = "%s :" % lfn.ljust( 100 )
for se in ses:
outStr = "%s %s" % ( outStr, se.ljust( 15 ) )
print outStr
def do_addFile( self, args ):
"""Add new files to transformation DB
usage: addFile <lfn> [lfn]
"""
argss = args.split()
if not len( argss ) > 0:
print "no files supplied"
return
lfnDict = {}
for lfn in argss:
lfnDict[lfn] = {'PFN':'IGNORED-PFN', 'SE':'IGNORED-SE', 'Size':0, 'GUID':'IGNORED-GUID',
'Checksum':'IGNORED-CHECKSUM'}
res = self.server.addFile( lfnDict, force = True )
if not res['OK']:
print "failed to add any files: %s" % res['Message']
return
for lfn in sorted( res['Value']['Failed'].keys() ):
error = res['Value']['Failed'][lfn]
print "failed to add %s: %s" % ( lfn, error )
for lfn in sorted( res['Value']['Successful'].keys() ):
print "added %s" % lfn
def do_removeFile( self, args ):
"""Remove file from transformation DB
usage: removeFile <lfn> [lfn]
"""
argss = args.split()
if not len( argss ) > 0:
print "no files supplied"
return
res = self.server.removeFile( argss )
if not res['OK']:
print "failed to remove any files: %s" % res['Message']
return
for lfn in sorted( res['Value']['Failed'].keys() ):
error = res['Value']['Failed'][lfn]
print "failed to remove %s: %s" % ( lfn, error )
for lfn in sorted( res['Value']['Successful'].keys() ):
print "removed %s" % lfn
def do_addReplica( self, args ):
""" Add new replica to the transformation DB
usage: addReplica <lfn> <se>
"""
argss = args.split()
if not len( argss ) == 2:
print "no file info supplied"
return
lfn = argss[0]
se = argss[1]
lfnDict = {}
lfnDict[lfn] = {'PFN':'IGNORED-PFN', 'SE':se, 'Size':0, 'GUID':'IGNORED-GUID', 'Checksum':'IGNORED-CHECKSUM'}
res = self.server.addReplica( lfnDict, force = True )
if not res['OK']:
print "failed to add replica: %s" % res['Message']
return
for lfn in sorted( res['Value']['Failed'].keys() ):
error = res['Value']['Failed'][lfn]
print "failed to add replica: %s" % ( error )
for lfn in sorted( res['Value']['Successful'].keys() ):
print "added %s" % lfn
def do_removeReplica( self, args ):
"""Remove replica from the transformation DB
usage: removeReplica <lfn> <se>
"""
argss = args.split()
if not len( argss ) == 2:
print "no file info supplied"
return
lfn = argss[0]
se = argss[1]
lfnDict = {}
lfnDict[lfn] = {'PFN':'IGNORED-PFN', 'SE':se, 'Size':0, 'GUID':'IGNORED-GUID', 'Checksum':'IGNORED-CHECKSUM'}
res = self.server.removeReplica( lfnDict )
if not res['OK']:
print "failed to remove replica: %s" % res['Message']
return
for lfn in sorted( res['Value']['Failed'].keys() ):
error = res['Value']['Failed'][lfn]
print "failed to remove replica: %s" % ( error )
for lfn in sorted( res['Value']['Successful'].keys() ):
print "removed %s" % lfn
def do_setReplicaStatus( self, args ):
"""Set replica status, usually used to mark a replica Problematic
usage: setReplicaStatus <lfn> <status> <se>
"""
argss = args.split()
if not len( argss ) > 2:
print "no file info supplied"
return
lfn = argss[0]
status = argss[1]
se = argss[2]
lfnDict = {}
lfnDict[lfn] = {'Status':status, 'PFN':'IGNORED-PFN', 'SE':se, 'Size':0, 'GUID':'IGNORED-GUID', 'Checksum':'IGNORED-CHECKSUM'}
res = self.server.setReplicaStatus( lfnDict )
if not res['OK']:
print "failed to set replica status: %s" % res['Message']
return
for lfn in sorted( res['Value']['Failed'].keys() ):
error = res['Value']['Failed'][lfn]
print "failed to set replica status: %s" % ( error )
for lfn in sorted( res['Value']['Successful'].keys() ):
print "updated replica status %s" % lfn
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
tc = TransformationClient()
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
fc = FileCatalogClient()
fmeta = {}
trans = None
info = []
if clip.prodid:
res = tc.getTransformation(clip.prodid)
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
trans = res['Value']
res = tc.getTransformationInputDataQuery( clip.prodid )
if res['OK']:
trans['InputDataQuery'] = res['Value']
res = tc.getAdditionalParameters ( clip.prodid )
if res['OK']:
trans['AddParams'] = res['Value']
#do something with transf
res = fc.findDirectoriesByMetadata({'ProdID':clip.prodid})
if res['OK']:
if len(res['Value'].values()):
gLogger.verbose("Found some directory matching the metadata")
for dirs in res['Value'].values():
res = fc.getDirectoryMetadata(dirs)
if res['OK']:
fmeta.update(res['Value'])
else:
def _getInfo():
"""gets info about transformation"""
clip = _Params()
clip.registerSwitches()
Script.parseCommandLine()
if not clip.prodid and not clip.filename:
Script.showHelp()
dexit(1)
from DIRAC import gLogger
import os
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
tc = TransformationClient()
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
fc = FileCatalogClient()
fmeta = {}
trans = None
info = []
if clip.prodid:
res = tc.getTransformation(clip.prodid)
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
trans = res['Value']
res = tc.getTransformationInputDataQuery( clip.prodid )
if res['OK']:
trans['InputDataQuery'] = res['Value']
res = tc.getAdditionalParameters ( clip.prodid )
if res['OK']:
trans['AddParams'] = res['Value']
#do something with transf
res1 = fc.findDirectoriesByMetadata({'ProdID':clip.prodid})
if res1['OK'] and len(res1['Value'].values()):
gLogger.verbose("Found %i directory matching the metadata" % len(res1['Value'].values()) )
for dirs in res1['Value'].values():
res = fc.getDirectoryUserMetadata(dirs)
if res['OK']:
fmeta.update(res['Value'])
else:
gLogger.error("Failed to get metadata for %s, SKIPPING" % dirs)
continue
res = fc.listDirectory(dirs)
if not res['OK']:
continue
content = res['Value']['Successful'][dirs]
if content["Files"]:
for f_ex in content["Files"].keys():
res = fc.getFileUserMetadata(f_ex)
if res['OK']:
fmeta.update(res['Value'])
break
#here we have trans and fmeta
info.append("")
info.append("Production %s has the following parameters:" % trans['TransformationID'])
info.extend(_createTransfoInfo(trans))
if fmeta:
info.append('The files created by this production have the following metadata:')
info.extend(_createFileInfo(fmeta))
info.append("It's possible that some meta data was not brought back,")
info.append("in particular file level metadata, so check some individual files")
if clip.filename:
pid = ""
if clip.filename.count("/"):
fpath = os.path.dirname(clip.filename)
res = fc.getDirectoryUserMetadata(fpath)
if not res['OK']:
gLogger.error(res['Message'])
dexit(0)
fmeta.update(res['Value'])
res = fc.getFileUserMetadata(clip.filename)
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
fmeta.update(res['Value'])
if 'ProdID' in fmeta:
pid = str(fmeta['ProdID'])
res = fc.getFileAncestors([clip.filename], 1)
if res["OK"]:
for dummy_lfn,ancestorsDict in res['Value']['Successful'].items():
if ancestorsDict.keys():
fmeta["Ancestors"] = ancestorsDict.keys()
res = fc.getFileDescendents([clip.filename], 1)
if res["OK"]:
for dummy_lfn,descendDict in res['Value']['Successful'].items():
if descendDict.keys():
fmeta['Descendants'] = descendDict.keys()
else:
ext = clip.filename.split(".")[-1]
fitems = []
for i in clip.filename.split('.')[:-1]:
fitems.extend(i.split('_'))
pid = ''
if ext == 'stdhep':
pid = fitems[fitems.index('gen')+1]
if ext == 'slcio':
if 'rec' in fitems:
pid = fitems[fitems.index('rec')+1]
elif 'dst' in fitems:
pid = fitems[fitems.index('dst')+1]
elif 'sim' in fitems:
pid = fitems[fitems.index('sim')+1]
else:
gLogger.error("This file does not follow the ILCDIRAC production conventions!")
gLogger.error("Please specify a prod ID directly or check the file.")
dexit(0)
if not pid:
gLogger.error("This file does not follow the ILCDIRAC production conventions!")
gLogger.error("Please specify a prod ID directly or check the file.")
dexit(0)
#as task follows the prod id, to get it we need
tid = fitems[fitems.index(pid)+1]
last_folder = str(int(tid)/1000).zfill(3)
res = fc.findDirectoriesByMetadata({'ProdID':int(pid)})
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
dir_ex = res['Value'].values()[0]
fpath = ""
if int(dir_ex.split("/")[-1]) == int(pid):
fpath = dir_ex+last_folder+"/"
elif int(dir_ex.split("/")[-2]) == int(pid):
fpath = "/".join(dir_ex.split('/')[:-2])+"/"+pid.zfill(8)+"/"+last_folder+"/"
else:
gLogger.error('Path does not follow conventions, will not get file family')
if fpath:
fpath += clip.filename
res = fc.getFileAncestors([fpath], 1)
if res["OK"]:
for dummy_lfn,ancestorsDict in res['Value']['Successful'].items():
fmeta["Ancestors"] = ancestorsDict.keys()
res = fc.getFileDescendents([fpath], 1)
if res["OK"]:
for dummy_lfn,descendDict in res['Value']['Successful'].items():
fmeta['Descendants'] = descendDict.keys()
res = fc.getDirectoryUserMetadata(dir_ex)
if not res['OK']:
gLogger.error(res['Message'])
else:
fmeta.update(res['Value'])
res = tc.getTransformation(pid)
if not res['OK']:
gLogger.error(res['Message'])
gLogger.error('Will proceed anyway')
else:
trans = res['Value']
res = tc.getTransformationInputDataQuery( pid )
if res['OK']:
trans['InputDataQuery'] = res['Value']
res = tc.getAdditionalParameters ( pid )
if res['OK']:
trans['AddParams'] = res['Value']
info.append("")
info.append("Input file has the following properties:")
info.extend(_createFileInfo(fmeta))
info.append("")
info.append('It was created with the production %s:' % pid)
if trans:
info.extend(_createTransfoInfo(trans))
gLogger.notice("\n".join(info))
dexit(0)
def _getInfo():
"""gets info about transformation"""
clip = _Params()
clip.registerSwitches()
Script.parseCommandLine()
if not clip.prodid and not clip.filename:
Script.showHelp()
dexit(1)
from DIRAC import gLogger
import os
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
tc = TransformationClient()
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
fc = FileCatalogClient()
fmeta = {}
trans = None
info = []
if clip.prodid:
res = tc.getTransformation(clip.prodid)
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
trans = res['Value']
res = tc.getTransformationInputDataQuery( clip.prodid )
if res['OK']:
trans['InputDataQuery'] = res['Value']
res = tc.getAdditionalParameters ( clip.prodid )
if res['OK']:
trans['AddParams'] = res['Value']
#do something with transf
res1 = fc.findDirectoriesByMetadata({'ProdID':clip.prodid})
if res1['OK'] and len(res1['Value'].values()):
gLogger.verbose("Found %i directory matching the metadata" % len(res1['Value'].values()) )
for dirs in res1['Value'].values():
res = fc.getDirectoryUserMetadata(dirs)
if res['OK']:
fmeta.update(res['Value'])
else:
gLogger.error("Failed to get metadata for %s, SKIPPING" % dirs)
continue
res = fc.listDirectory(dirs)
if not res['OK']:
continue
content = res['Value']['Successful'][dirs]
if content["Files"]:
for f_ex in content["Files"].keys():
res = fc.getFileUserMetadata(f_ex)
if res['OK']:
fmeta.update(res['Value'])
break
#here we have trans and fmeta
info.append("")
info.append("Production %s has the following parameters:" % trans['TransformationID'])
info.extend(_createTransfoInfo(trans))
if fmeta:
info.append('The files created by this production have the following metadata:')
info.extend(_createFileInfo(fmeta))
info.append("It's possible that some meta data was not brought back,")
info.append("in particular file level metadata, so check some individual files")
if clip.filename:
pid = ""
if clip.filename.count("/"):
fpath = os.path.dirname(clip.filename)
res = fc.getDirectoryUserMetadata(fpath)
if not res['OK']:
gLogger.error(res['Message'])
dexit(0)
fmeta.update(res['Value'])
res = fc.getFileUserMetadata(clip.filename)
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
fmeta.update(res['Value'])
if 'ProdID' in fmeta:
pid = str(fmeta['ProdID'])
res = fc.getFileAncestors([clip.filename], 1)
if res["OK"]:
for dummy_lfn,ancestorsDict in res['Value']['Successful'].items():
if ancestorsDict.keys():
fmeta["Ancestors"] = ancestorsDict.keys()
res = fc.getFileDescendents([clip.filename], 1)
if res["OK"]:
for dummy_lfn,descendDict in res['Value']['Successful'].items():
if descendDict.keys():
fmeta['Descendants'] = descendDict.keys()
else:
ext = clip.filename.split(".")[-1]
fitems = []
for i in clip.filename.split('.')[:-1]:
fitems.extend(i.split('_'))
pid = ''
if ext == 'stdhep':
pid = fitems[fitems.index('gen')+1]
if ext == 'slcio':
if 'rec' in fitems:
pid = fitems[fitems.index('rec')+1]
elif 'dst' in fitems:
pid = fitems[fitems.index('dst')+1]
elif 'sim' in fitems:
pid = fitems[fitems.index('sim')+1]
else:
gLogger.error("This file does not follow the ILCDIRAC production conventions!")
gLogger.error("Please specify a prod ID directly or check the file.")
dexit(0)
if not pid:
gLogger.error("This file does not follow the ILCDIRAC production conventions!")
gLogger.error("Please specify a prod ID directly or check the file.")
dexit(0)
#as task follows the prod id, to get it we need
tid = fitems[fitems.index(pid)+1]
last_folder = str(int(tid)/1000).zfill(3)
res = fc.findDirectoriesByMetadata({'ProdID':int(pid)})
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
dir_ex = res['Value'].values()[0]
fpath = ""
if int(dir_ex.split("/")[-1]) == int(pid):
fpath = dir_ex+last_folder+"/"
elif int(dir_ex.split("/")[-2]) == int(pid):
fpath = "/".join(dir_ex.split('/')[:-2])+"/"+pid.zfill(8)+"/"+last_folder+"/"
else:
gLogger.error('Path does not follow conventions, will not get file family')
if fpath:
fpath += clip.filename
res = fc.getFileAncestors([fpath], 1)
if res["OK"]:
for dummy_lfn,ancestorsDict in res['Value']['Successful'].items():
fmeta["Ancestors"] = ancestorsDict.keys()
res = fc.getFileDescendents([fpath], 1)
if res["OK"]:
for dummy_lfn,descendDict in res['Value']['Successful'].items():
fmeta['Descendants'] = descendDict.keys()
res = fc.getDirectoryUserMetadata(dir_ex)
if not res['OK']:
gLogger.error(res['Message'])
else:
fmeta.update(res['Value'])
res = tc.getTransformation(pid)
if not res['OK']:
gLogger.error(res['Message'])
gLogger.error('Will proceed anyway')
else:
trans = res['Value']
res = tc.getTransformationInputDataQuery( pid )
if res['OK']:
trans['InputDataQuery'] = res['Value']
res = tc.getAdditionalParameters ( pid )
if res['OK']:
trans['AddParams'] = res['Value']
info.append("")
info.append("Input file has the following properties:")
info.extend(_createFileInfo(fmeta))
info.append("")
info.append('It was created with the production %s:' % pid)
if trans:
info.extend(_createTransfoInfo(trans))
gLogger.notice("\n".join(info))
dexit(0)
def _getProductionSummary():
clip = _Params()
clip.registerSwitch()
Script.parseCommandLine()
from ILCDIRAC.Core.Utilities.HTML import Table
from ILCDIRAC.Core.Utilities.ProcessList import ProcessList
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
from DIRAC import gConfig, gLogger
prod = clip.prod
full_detail = clip.full_det
fc = FileCatalogClient()
processlist = gConfig.getValue('/LocalSite/ProcessListPath')
prl = ProcessList(processlist)
processesdict = prl.getProcessesDict()
trc = TransformationClient()
prodids = []
if not prod:
conddict = {}
conddict['Status'] = clip.statuses
if clip.ptypes:
conddict['Type'] = clip.ptypes
res = trc.getTransformations( conddict )
if res['OK']:
for transfs in res['Value']:
prodids.append(transfs['TransformationID'])
else:
prodids.extend(prod)
metadata = []
gLogger.info("Will run on prods %s" % str(prodids))
for prodID in prodids:
if prodID<clip.minprod:
continue
meta = {}
meta['ProdID']=prodID
res = trc.getTransformation(str(prodID))
if not res['OK']:
gLogger.error("Error getting transformation %s" % prodID )
continue
prodtype = res['Value']['Type']
proddetail = res['Value']['Description']
if prodtype == 'MCReconstruction' or prodtype == 'MCReconstruction_Overlay' :
meta['Datatype']='DST'
elif prodtype == 'MCGeneration':
meta['Datatype']='gen'
elif prodtype == 'MCSimulation':
meta['Datatype']='SIM'
elif prodtype in ['Split','Merge']:
gLogger.warn("Invalid query for %s productions" % prodtype)
continue
else:
gLogger.error("Unknown production type %s"% prodtype)
continue
res = fc.findFilesByMetadata(meta)
if not res['OK']:
gLogger.error(res['Message'])
continue
lfns = res['Value']
nb_files = len(lfns)
path = ""
if not len(lfns):
gLogger.warn("No files found for prod %s" % prodID)
continue
path = os.path.dirname(lfns[0])
res = fc.getDirectoryUserMetadata(path)
if not res['OK']:
gLogger.warn('No meta data found for %s' % path)
continue
dirmeta = {}
dirmeta['proddetail'] = proddetail
dirmeta['prodtype'] = prodtype
dirmeta['nb_files']=nb_files
dirmeta.update(res['Value'])
lumi = 0.
nbevts = 0
addinfo = None
files = 0
xsec = 0.0
if not full_detail:
lfn = lfns[0]
info = _getFileInfo(lfn)
nbevts = info[1]*len(lfns)
lumi = info[0]*len(lfns)
addinfo = info[2]
if 'xsection' in addinfo:
if 'sum' in addinfo['xsection']:
if 'xsection' in addinfo['xsection']['sum']:
xsec += addinfo['xsection']['sum']['xsection']
files += 1
else:
for lfn in lfns:
info = _getFileInfo(lfn)
lumi += info[0]
nbevts += info[1]
addinfo = info[2]
if 'xsection' in addinfo:
if 'sum' in addinfo['xsection']:
if 'xsection' in addinfo['xsection']['sum']:
xsec += addinfo['xsection']['sum']['xsection']
files += 1
if not lumi:
xsec = 0
files = 0
depthDict = {}
depSet = set()
res = fc.getFileAncestors(lfns,[1,2,3,4])
temp_ancestorlist = []
if res['OK']:
for lfn,ancestorsDict in res['Value']['Successful'].items():
for ancestor,dep in ancestorsDict.items():
depthDict.setdefault(dep,[])
if ancestor not in temp_ancestorlist:
depthDict[dep].append(ancestor)
depSet.add(dep)
temp_ancestorlist.append(ancestor)
depList = list(depSet)
depList.sort()
for ancestor in depthDict[depList[-1]]:
info = _getFileInfo(ancestor)
lumi += info[0]
addinfo = info[2]
if 'xsection' in addinfo:
if 'sum' in addinfo['xsection']:
if 'xsection' in addinfo['xsection']['sum']:
xsec += addinfo['xsection']['sum']['xsection']
files += 1
if xsec and files:
xsec /= files
dirmeta['CrossSection']=xsec
else:
dirmeta['CrossSection']=0.0
if nbevts:
dirmeta['NumberOfEvents']=nbevts
#if not lumi:
# dirmeta['Luminosity']=0
# dirmeta['CrossSection']=0
#else:
# if nbevts:
# dirmeta['CrossSection']=nbevts/lumi
# else:
# dirmeta['CrossSection']=0
#if addinfo:
# if 'xsection' in addinfo:
# if 'sum' in addinfo['xsection']:
# if 'xsection' in addinfo['xsection']['sum']:
# dirmeta['CrossSection']=addinfo['xsection']['sum']['xsection']
if 'NumberOfEvents' not in dirmeta:
dirmeta['NumberOfEvents']=0
#print processesdict[dirmeta['EvtType']]
dirmeta['detail']=''
if dirmeta['EvtType'] in processesdict:
if 'Detail' in processesdict[dirmeta['EvtType']]:
detail = processesdict[dirmeta['EvtType']]['Detail']
else:
detail=dirmeta['EvtType']
if not prodtype == 'MCGeneration':
res = trc.getTransformationInputDataQuery(str(prodID))
if res['OK']:
if 'ProdID' in res['Value']:
dirmeta['MomProdID']=res['Value']['ProdID']
if 'MomProdID' not in dirmeta:
dirmeta['MomProdID']=0
dirmeta['detail']= _translate(detail)
metadata.append(dirmeta)
detectors = {}
detectors['ILD'] = {}
corres = {"MCGeneration":'gen',"MCSimulation":'SIM',"MCReconstruction":"REC","MCReconstruction_Overlay":"REC"}
detectors['ILD']['SIM'] = []
detectors['ILD']['REC'] = []
detectors['SID'] = {}
detectors['SID']['SIM'] = []
detectors['SID']['REC'] = []
detectors['sid'] = {}
detectors['sid']['SIM'] = []
detectors['sid']['REC'] = []
detectors['gen']=[]
for channel in metadata:
if 'DetectorType' not in channel:
detectors['gen'].append((channel['detail'],
channel['Energy'],
channel['ProdID'],
channel['nb_files'],
channel['NumberOfEvents']/channel['nb_files'],
channel['NumberOfEvents'],
channel['CrossSection'],str(channel['proddetail'])))
else:
if not channel['DetectorType'] in detectors:
gLogger.error("This is unknown detector", channel['DetectorType'])
continue
detectors[channel['DetectorType']][corres[channel['prodtype']]].append((channel['detail'],
channel['Energy'],
channel['DetectorType'],
channel['ProdID'],
channel['nb_files'],
channel['NumberOfEvents']/channel['nb_files'],
channel['NumberOfEvents'],
channel['CrossSection'],
channel['MomProdID'],
str(channel['proddetail'])))
with open("tables.html","w") as of:
of.write("""<!DOCTYPE html>
<html>
<head>
<title> Production summary </title>
</head>
<body>
""")
if len(detectors['gen']):
of.write("<h1>gen prods</h1>\n")
table = Table(header_row = ('Channel', 'Energy','ProdID','Tasks','Average Evts/task','Statistics','Cross Section (fb)','Comment'))
for item in detectors['gen']:
table.rows.append( item )
of.write(str(table))
gLogger.info("Gen prods")
gLogger.info(str(table))
if len(detectors['ILD']):
of.write("<h1>ILD prods</h1>\n")
for ptype in detectors['ILD'].keys():
if len(detectors['ILD'][ptype]):
of.write("<h2>%s</h2>\n"%ptype)
table = Table(header_row = ('Channel', 'Energy','Detector','ProdID','Number of Files','Events/File','Statistics','Cross Section (fb)','Origin ProdID','Comment'))
for item in detectors['ILD'][ptype]:
table.rows.append( item )
of.write(str(table))
gLogger.info("ILC CDR prods %s" % ptype)
gLogger.info(str(table))
if len(detectors['SID']):
of.write("<h1>SID prods</h1>\n")
for ptype in detectors['SID'].keys():
if len(detectors['SID'][ptype]):
of.write("<h2>%s</h2>\n"%ptype)
table = Table(header_row = ('Channel', 'Energy','Detector','ProdID','Number of Files','Events/File','Statistics','Cross Section (fb)','Origin ProdID','Comment'))
for item in detectors['SID'][ptype]:
table.rows.append( item )
of.write(str(table))
gLogger.info("SID CDR prods %s"%ptype)
gLogger.info(str(table))
if len(detectors['sid']):
of.write("<h1>sid dbd prods</h1>\n")
for ptype in detectors['sid'].keys():
if len(detectors['sid'][ptype]):
of.write("<h2>%s</h2>\n"%ptype)
table = Table(header_row = ('Channel', 'Energy','Detector','ProdID','Number of Files','Events/File','Statistics','Cross Section (fb)','Origin ProdID','Comment'))
for item in detectors['sid'][ptype]:
table.rows.append( item )
of.write(str(table))
gLogger.info("sid DBD prods %s"%ptype)
gLogger.info(str(table))
of.write("""
</body>
</html>
""")
gLogger.notice("Check ./tables.html in any browser for the results")
dexit(0)
class InputDataAgent(AgentModule):
#############################################################################
def initialize(self):
""" Make the necessary initializations """
self.fileLog = {}
self.timeLog = {}
self.fullTimeLog = {}
self.pollingTime = self.am_getOption('PollingTime',120)
self.fullUpdatePeriod = self.am_getOption('FullUpdatePeriod',86400)
gMonitor.registerActivity("Iteration","Agent Loops",AGENT_NAME,"Loops/min",gMonitor.OP_SUM)
self.transClient = TransformationClient('TransformationDB')
self.metadataClient = FileCatalogClient()
return S_OK()
##############################################################################
def execute(self):
""" Main execution method
"""
gMonitor.addMark('Iteration',1)
# Get all the transformations
result = self.transClient.getTransformations(condDict={'Status':'Active'})
activeTransforms = []
if not result['OK']:
gLogger.error("InputDataAgent.execute: Failed to get transformations.", result['Message'])
return S_OK()
# Process each transformation
for transDict in result['Value']:
transID = long(transDict['TransformationID'])
res = self.transClient.getTransformationInputDataQuery(transID)
if not res['OK']:
if res['Message'] == 'No InputDataQuery found for transformation':
gLogger.info("InputDataAgent.execute: No input data query found for transformation %d" % transID)
else:
gLogger.error("InputDataAgent.execute: Failed to get input data query for %d" % transID, res['Message'])
continue
inputDataQuery = res['Value']
# Determine the correct time stamp to use for this transformation
if self.timeLog.has_key(transID):
if self.fullTimeLog.has_key(transID):
# If it is more than a day since the last reduced query, make a full query just in case
if (datetime.datetime.utcnow() - self.fullTimeLog[transID]) < datetime.timedelta(seconds=self.fullUpdatePeriod):
timeStamp = self.timeLog[transID]
inputDataQuery['StartDate'] = (timeStamp - datetime.timedelta(seconds=10)).strftime('%Y-%m-%d %H:%M:%S')
else:
self.fullTimeLog[transID] = datetime.datetime.utcnow()
self.timeLog[transID] = datetime.datetime.utcnow()
if not self.fullTimeLog.has_key(transID):
self.fullTimeLog[transID] = datetime.datetime.utcnow()
# Perform the query to the metadata catalog
gLogger.verbose("Using input data query for transformation %d: %s" % (transID,str(inputDataQuery)))
start = time.time()
result = self.metadataClient.findFilesByMetadata(inputDataQuery)
rtime = time.time()-start
gLogger.verbose("Metadata catalog query time: %.2f seconds." % (rtime))
if not result['OK']:
gLogger.error("InputDataAgent.execute: Failed to get response from the metadata catalog", result['Message'])
continue
lfnList = result['Value']
# Check if the number of files has changed since the last cycle
nlfns = len(lfnList)
gLogger.info("%d files returned for transformation %d from the metadata catalog" % (nlfns,int(transID)) )
if self.fileLog.has_key(transID):
if nlfns == self.fileLog[transID]:
gLogger.verbose('No new files in metadata catalog since last check')
self.fileLog[transID] = nlfns
# Add any new files to the transformation
addedLfns = []
if lfnList:
gLogger.verbose('Processing %d lfns for transformation %d' % (len(lfnList),transID) )
# Add the files to the transformation
gLogger.verbose('Adding %d lfns for transformation %d' % (len(lfnList),transID) )
result = self.transClient.addFilesToTransformation(transID,sortList(lfnList))
if not result['OK']:
gLogger.warn("InputDataAgent.execute: failed to add lfns to transformation", result['Message'])
self.fileLog[transID] = 0
else:
if result['Value']['Failed']:
for lfn,error in res['Value']['Failed'].items():
gLogger.warn("InputDataAgent.execute: Failed to add %s to transformation" % lfn,error)
if result['Value']['Successful']:
for lfn,status in result['Value']['Successful'].items():
if status == 'Added':
addedLfns.append(lfn)
gLogger.info("InputDataAgent.execute: Added %d files to transformation" % len(addedLfns))
return S_OK()
