def submit_TS_no_MQ(job, infileList):
""" Create a transformation executing the job workflow
"""
t = Transformation()
tc = TransformationClient()
t.setType("DataReprocessing")
t.setDescription("Runs ImageExtractor analysis for array HB9 SCT")
t.setLongDescription(
"merge_simtel, ImageExtractor DL0->DL1 conversion for HB9 SCT")
t.setGroupSize(5)
t.setBody(job.workflow.toXML())
res = t.addTransformation() # Transformation is created here
if not res['OK']:
Script.gLogger.error(res['Message'])
DIRAC.exit(-1)
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
Script.gLogger.notice('Adding %s files to transformation' %
len(infileList))
tc.addFilesToTransformation(transID['Value'], infileList)
return res
def OldSubmitTS(job, infileList):
""" Create a transformation executing the job workflow """
t = Transformation()
tc = TransformationClient()
t.setType("DataReprocessing")
t.setDescription("Runs EvnDisp analysis for array HB9 SCT")
t.setLongDescription(
"merge_simtel, evndisp converter and evndisp analysis for HB9 SCT"
) # mandatory
t.setGroupSize(5)
t.setBody(job.workflow.toXML())
res = t.addTransformation() # Transformation is created here
if not res['OK']:
print res['Message']
DIRAC.exit(-1)
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
print('Adding %s files to transformation' % len(infileList))
tc.addFilesToTransformation(transID['Value'],
infileList) # Files added here
return res
def ReproTSExample( args = None ) :
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
from CTADIRAC.Interfaces.API.ReproTSJob import ReproTSJob
if (len(args) != 2):
Script.gLogger.notice('Wrong number of arguments')
Script.showHelp()
infile = args[0]
f = open(infile,'r')
infileList = []
for line in f:
infile = line.strip()
if line!="\n":
infileList.append(infile)
if args[1] not in ['STD','NSBX3','SCMST','SCSST','4MSST','ASTRI','NORTH']:
Script.gLogger.notice('reprocessing configuration incorrect:',args[1])
Script.showHelp()
simtelArrayConfig = args[1]
j = ReproTSJob()
j.setVersion('prod-2_15122013')
j.setParameters(['fileCatalog.cfg','-S',simtelArrayConfig])
j.setInputSandbox( ['LFN:/vo.cta.in2p3.fr/MC/PROD2/CFG_rev6956/prod2_cfg.tar.gz'])
j.setOutputSandbox( ['*.log','applicationLog.txt'])
j.setCPUTime(200000)
t = Transformation( )
tc = TransformationClient( )
t.setTransformationName("Reprotest1") # This must vary
#t.setTransformationGroup("Group1")
t.setType("DataReprocessing")
t.setDescription("simtel repro example")
t.setLongDescription( "simtel reprocessing" ) #mandatory
t.setBody ( j.workflow.toXML() )
t.addTransformation() #transformation is created here
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
tc.addFilesToTransformation(transID['Value'],infileList) # Files added here
def submit_trans(job, infileList, trans_name, group_size):
""" Create a transformation executing the job workflow
"""
DIRAC.gLogger.notice('submit_trans : %s' % trans_name)
# Initialize JOB_ID
job.workflow.addParameter(
Parameter("JOB_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
trans = Transformation()
trans.setTransformationName(trans_name) # this must be unique
trans.setType("DataReprocessing")
trans.setDescription("Simtel TS example")
trans.setLongDescription("Simtel tel_sim") # mandatory
trans.setBody(job.workflow.toXML())
trans.setGroupSize(group_size)
res = trans.addTransformation() # transformation is created here
if not res['OK']:
return res
trans.setStatus("Active")
trans.setAgentType("Automatic")
# add 10*group_size files to transformation (to have the first 10 jobs)
trans_id = trans.getTransformationID()
trans_client = TransformationClient()
res = trans_client.addFilesToTransformation(trans_id['Value'],
infileList[:10 * group_size])
return res
def DataReplicaRemovalTSExample(args=None):
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
if (len(args) != 1):
Script.gLogger.notice('Wrong number of arguments')
Script.showHelp()
infile = args[0]
f = open(infile, 'r')
infileList = []
for line in f:
infile = line.strip()
if line != "\n":
infileList.append(infile)
t = Transformation()
tc = TransformationClient()
t.setType("Removal")
t.setPlugin("Broadcast") # Mandatory for ReplicaRemoval
t.setSourceSE(['CC-IN2P3-Tape'
]) # A list of SE where at least 1 SE is the valid one
t.setTargetSE([
'CEA-Disk', 'LAPP-Disk', 'DESY-ZN-Disk', 'CC-IN2P3-Disk',
'CYF-STORM-Disk'
]) # A list of SE where at least 1 SE is the valid one
t.setDescription("AarProd2 ReplicaRemoval")
t.setLongDescription("AarProd2 ReplicaRemoval") # Mandatory
t.setGroupSize(
100
) # Here you specify how many files should be grouped within the same request, e.g. 100
t.setBody("Removal;RemoveReplica"
) # Mandatory (the default is a ReplicateAndRegister operation)
t.addTransformation() # Transformation is created here
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
tc.addFilesToTransformation(transID['Value'],
infileList) # Files are added here
def DataReplicationTSExample(args=None):
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
if (len(args) != 1):
Script.gLogger.notice('Wrong number of arguments')
Script.showHelp()
infile = args[0]
f = open(infile, 'r')
infileList = []
for line in f:
infile = line.strip()
if line != "\n":
infileList.append(infile)
t = Transformation()
tc = TransformationClient()
t.setTransformationName("DM_Replication") # This must be unique
#t.setTransformationGroup("Group1")
t.setType("Replication")
t.setSourceSE(['CYF-STORM-Disk', 'DESY-ZN-Disk'
]) # A list of SE where at least 1 SE is the valid one
t.setTargetSE(['CEA-Disk'])
t.setDescription("corsika Replication")
t.setLongDescription("corsika Replication") # Mandatory
t.setGroupSize(
1
) # Here you specify how many files should be grouped within the same request, e.g. 100
t.setPlugin("Broadcast") # Mandatory for replication
t.addTransformation() # Transformation is created here
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
tc.addFilesToTransformation(transID['Value'],
infileList) # Files are added here
def DataRemovalTSExample(args=None):
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
if (len(args) != 1):
Script.gLogger.notice('Wrong number of arguments')
Script.showHelp()
infile = args[0]
f = open(infile, 'r')
infileList = []
for line in f:
infile = line.strip()
if line != "\n":
infileList.append(infile)
t = Transformation()
tc = TransformationClient()
t.setTransformationName("DM_Removal26") # Must be unique
#t.setTransformationGroup("Group1")
t.setType("Removal")
t.setPlugin("Standard") # Not needed. The default is 'Standard'
t.setDescription("corsika Removal")
t.setLongDescription("corsika Removal") # Mandatory
t.setGroupSize(
1
) # Here you specify how many files should be grouped within the same request, e.g. 100
t.setBody("Removal;RemoveFile"
) # Mandatory (the default is a ReplicateAndRegister operation)
t.addTransformation() # Transformation is created here
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
tc.addFilesToTransformation(transID['Value'],
infileList) # Files are added here
def submitTS( job, infileList ):
""" Create a transformation executing the job workflow """
t = Transformation()
tc = TransformationClient()
t.setType( "DataReprocessing" )
t.setDescription( "Chimp example" )
t.setLongDescription( "Chimp analysis" ) # mandatory
t.setGroupSize(1)
t.setBody ( job.workflow.toXML() )
res = t.addTransformation() # Transformation is created here
if not res['OK']:
print res['Message']
DIRAC.exit( -1 )
t.setStatus( "Active" )
t.setAgentType( "Automatic" )
transID = t.getTransformationID()
tc.addFilesToTransformation( transID['Value'], infileList ) # Files added here
return res
def submitTS( job, infileList ):
""" Create a transformation executing the job workflow """
t = Transformation()
tc = TransformationClient()
t.setType( "SimtelMerging" )
t.setDescription( "Runs merge_simtel for array 3HB89" )
t.setLongDescription( "Merging array 3HB89 analysis" ) # mandatory
t.setGroupSize(5)
t.setBody ( job.workflow.toXML() )
res = t.addTransformation() # Transformation is created here
if not res['OK']:
print res['Message']
DIRAC.exit( -1 )
t.setStatus( "Active" )
t.setAgentType( "Automatic" )
transID = t.getTransformationID()
print('Adding %s files to transformation'%len(infileList))
tc.addFilesToTransformation( transID['Value'], infileList ) # Files added here
return res
def main():
Script.parseCommandLine()
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
args = Script.getPositionalArgs()
if len(args) < 2:
Script.showHelp(exitCode=1)
# get arguments
inputFileName = args[1]
lfns = []
if os.path.exists(inputFileName):
inputFile = open(inputFileName, 'r')
string = inputFile.read()
inputFile.close()
lfns.extend([lfn.strip() for lfn in string.splitlines()])
else:
lfns.append(inputFileName)
tc = TransformationClient()
res = tc.addFilesToTransformation(args[0], lfns) # Files added here
if not res['OK']:
DIRAC.gLogger.error(res['Message'])
DIRAC.exit(2)
successfullyAdded = 0
alreadyPresent = 0
for lfn, message in res['Value']['Successful'].items():
if message == 'Added':
successfullyAdded += 1
elif message == 'Present':
alreadyPresent += 1
if successfullyAdded > 0:
DIRAC.gLogger.notice("Successfully added %d files" % successfullyAdded)
if alreadyPresent > 0:
DIRAC.gLogger.notice("Already present %d files" % alreadyPresent)
DIRAC.exit(0)
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument("TransID: transformation ID")
Script.registerArgument(("LFN: LFN", "FileName: file containing LFNs"))
Script.parseCommandLine()
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
tID, inputFileName = Script.getPositionalArgs(group=True)
# get arguments
lfns = []
if os.path.exists(inputFileName):
inputFile = open(inputFileName, "r")
string = inputFile.read()
inputFile.close()
lfns.extend([lfn.strip() for lfn in string.splitlines()])
else:
lfns.append(inputFileName)
tc = TransformationClient()
res = tc.addFilesToTransformation(tID, lfns) # Files added here
if not res["OK"]:
DIRAC.gLogger.error(res["Message"])
DIRAC.exit(2)
successfullyAdded = 0
alreadyPresent = 0
for lfn, message in res["Value"]["Successful"].items():
if message == "Added":
successfullyAdded += 1
elif message == "Present":
alreadyPresent += 1
if successfullyAdded > 0:
DIRAC.gLogger.notice("Successfully added %d files" % successfullyAdded)
if alreadyPresent > 0:
DIRAC.gLogger.notice("Already present %d files" % alreadyPresent)
DIRAC.exit(0)
def submit_trans(job, trans_name, infileList, group_size):
""" Create a transformation executing the job workflow
"""
DIRAC.gLogger.notice('submit_trans : %s' % trans_name)
# Initialize JOB_ID
job.workflow.addParameter(
Parameter("JOB_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
trans = Transformation()
trans.setTransformationName(trans_name) # this must be unique
trans.setType("DataReprocessing")
trans.setDescription("SimpleCtapipe example")
trans.setLongDescription(
"ctapipe classify, reconstruct and merge: calib_imgreco") # mandatory
trans.setBody(job.workflow.toXML())
trans.setGroupSize(group_size)
res = trans.addTransformation() # transformation is created here
if not res['OK']:
DIRAC.gLogger.error('Error creating transformation: ', res['Message'])
return res
transID = res['Value']
# Activate the transformation
trans.setStatus("Active")
trans.setAgentType("Automatic")
# Add files to the transformation
tc = TransformationClient()
res = tc.addFilesToTransformation(transID, infileList)
if not res['OK']:
DIRAC.gLogger.error('Error adding files to the transformation: ',
res['Message'])
return res
class InputDataAgent(AgentModule):
#############################################################################
def initialize(self):
""" Make the necessary initializations """
self.fileLog = {}
self.timeLog = {}
self.fullTimeLog = {}
self.pollingTime = self.am_getOption('PollingTime', 120)
self.fullUpdatePeriod = self.am_getOption('FullUpdatePeriod', 86400)
gMonitor.registerActivity("Iteration", "Agent Loops", AGENT_NAME,
"Loops/min", gMonitor.OP_SUM)
self.transClient = TransformationClient('TransformationDB')
self.metadataClient = FileCatalogClient()
return S_OK()
##############################################################################
def execute(self):
""" Main execution method
"""
gMonitor.addMark('Iteration', 1)
# Get all the transformations
result = self.transClient.getTransformations(
condDict={'Status': 'Active'})
activeTransforms = []
if not result['OK']:
gLogger.error(
"InputDataAgent.execute: Failed to get transformations.",
result['Message'])
return S_OK()
# Process each transformation
for transDict in result['Value']:
transID = long(transDict['TransformationID'])
res = self.transClient.getTransformationInputDataQuery(transID)
if not res['OK']:
if res['Message'] == 'No InputDataQuery found for transformation':
gLogger.info(
"InputDataAgent.execute: No input data query found for transformation %d"
% transID)
else:
gLogger.error(
"InputDataAgent.execute: Failed to get input data query for %d"
% transID, res['Message'])
continue
inputDataQuery = res['Value']
# Determine the correct time stamp to use for this transformation
if self.timeLog.has_key(transID):
if self.fullTimeLog.has_key(transID):
# If it is more than a day since the last reduced query, make a full query just in case
if (datetime.datetime.utcnow() -
self.fullTimeLog[transID]) < datetime.timedelta(
seconds=self.fullUpdatePeriod):
timeStamp = self.timeLog[transID]
inputDataQuery['StartDate'] = (
timeStamp - datetime.timedelta(seconds=10)
).strftime('%Y-%m-%d %H:%M:%S')
else:
self.fullTimeLog[transID] = datetime.datetime.utcnow()
self.timeLog[transID] = datetime.datetime.utcnow()
if not self.fullTimeLog.has_key(transID):
self.fullTimeLog[transID] = datetime.datetime.utcnow()
# Perform the query to the metadata catalog
gLogger.verbose(
"Using input data query for transformation %d: %s" %
(transID, str(inputDataQuery)))
start = time.time()
result = self.metadataClient.findFilesByMetadata(inputDataQuery)
rtime = time.time() - start
gLogger.verbose("Metadata catalog query time: %.2f seconds." %
(rtime))
if not result['OK']:
gLogger.error(
"InputDataAgent.execute: Failed to get response from the metadata catalog",
result['Message'])
continue
lfnList = result['Value']
# Check if the number of files has changed since the last cycle
nlfns = len(lfnList)
gLogger.info(
"%d files returned for transformation %d from the metadata catalog"
% (nlfns, int(transID)))
if self.fileLog.has_key(transID):
if nlfns == self.fileLog[transID]:
gLogger.verbose(
'No new files in metadata catalog since last check')
self.fileLog[transID] = nlfns
# Add any new files to the transformation
addedLfns = []
if lfnList:
gLogger.verbose('Processing %d lfns for transformation %d' %
(len(lfnList), transID))
# Add the files to the transformation
gLogger.verbose('Adding %d lfns for transformation %d' %
(len(lfnList), transID))
result = self.transClient.addFilesToTransformation(
transID, sortList(lfnList))
if not result['OK']:
gLogger.warn(
"InputDataAgent.execute: failed to add lfns to transformation",
result['Message'])
self.fileLog[transID] = 0
else:
if result['Value']['Failed']:
for lfn, error in res['Value']['Failed'].items():
gLogger.warn(
"InputDataAgent.execute: Failed to add %s to transformation"
% lfn, error)
if result['Value']['Successful']:
for lfn, status in result['Value']['Successful'].items(
):
if status == 'Added':
addedLfns.append(lfn)
gLogger.info(
"InputDataAgent.execute: Added %d files to transformation"
% len(addedLfns))
return S_OK()
DIRAC.exit( 1 )
# get arguments
inputFileName = args[1]
lfns = []
if os.path.exists( inputFileName ):
inputFile = open( inputFileName, 'r' )
string = inputFile.read()
inputFile.close()
lfns.extend( [ lfn.strip() for lfn in string.splitlines() ] )
else:
lfns.append( inputFileName )
tc = TransformationClient()
res = tc.addFilesToTransformation( args[0] , lfns ) # Files added here
if not res['OK']:
DIRAC.gLogger.error ( res['Message'] )
DIRAC.exit( 2 )
successfullyAdded = 0
alreadyPresent = 0
for lfn, message in res['Value']['Successful'].items():
if message == 'Added':
successfullyAdded += 1
elif message == 'Present':
alreadyPresent += 1
if successfullyAdded > 0:
DIRAC.gLogger.notice( "Successfully added %d files" % successfullyAdded )
'\ne.g: %s 381 Paranal_gamma_North.list' % Script.scriptName,
]))
Script.parseCommandLine()
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
args = Script.getPositionalArgs()
if (len(args) != 2):
Script.showHelp()
# get arguments
TransID = args[0]
infile = args[1]
f = open(infile, 'r')
infileList = []
for line in f:
infile = line.strip()
if line != "\n":
infileList.append(infile)
tc = TransformationClient()
res = tc.addFilesToTransformation(TransID, infileList) # Files added here
if not res['OK']:
DIRAC.gLogger.error(res['Message'])
DIRAC.exit(-1)
else:
DIRAC.exit(0)
class InputDataAgent(AgentModule):
def __init__(self, *args, **kwargs):
"""c'tor"""
AgentModule.__init__(self, *args, **kwargs)
self.fileLog = {}
self.timeLog = {}
self.fullTimeLog = {}
self.pollingTime = self.am_getOption("PollingTime", 120)
self.fullUpdatePeriod = self.am_getOption("FullUpdatePeriod", 86400)
self.refreshonly = self.am_getOption("RefreshOnly", False)
self.dateKey = self.am_getOption("DateKey", None)
self.transClient = TransformationClient()
self.metadataClient = FileCatalogClient()
self.transformationTypes = None
#############################################################################
def initialize(self):
"""Make the necessary initializations"""
agentTSTypes = self.am_getOption("TransformationTypes", [])
if agentTSTypes:
self.transformationTypes = sorted(agentTSTypes)
else:
dataProc = Operations().getValue("Transformations/DataProcessing",
["MCSimulation", "Merge"])
dataManip = Operations().getValue(
"Transformations/DataManipulation", ["Replication", "Removal"])
self.transformationTypes = sorted(dataProc + dataManip)
extendables = Operations().getValue(
"Transformations/ExtendableTransfTypes", [])
if extendables:
for extendable in extendables:
if extendable in self.transformationTypes:
self.transformationTypes.remove(extendable)
# This is because the Extendables do not use this Agent (have no Input data query)
return S_OK()
##############################################################################
def execute(self):
"""Main execution method"""
# Get all the transformations
result = self.transClient.getTransformations({
"Status":
"Active",
"Type":
self.transformationTypes
})
if not result["OK"]:
self.log.error(
"InputDataAgent.execute: Failed to get transformations.",
result["Message"])
return S_OK()
# Process each transformation
for transDict in result["Value"]:
transID = int(transDict["TransformationID"])
# res = self.transClient.getTransformationInputDataQuery( transID )
res = self.transClient.getTransformationMetaQuery(transID, "Input")
if not res["OK"]:
if cmpError(res, ENOENT):
self.log.info(
"InputDataAgent.execute: No input data query found for transformation",
transID)
else:
self.log.error(
"InputDataAgent.execute: Failed to get input data query",
"for %d: %s" % (transID, res["Message"]),
)
continue
inputDataQuery = res["Value"]
if self.refreshonly:
# Determine the correct time stamp to use for this transformation
if transID in self.timeLog:
if transID in self.fullTimeLog:
# If it is more than a day since the last reduced query, make a full query just in case
if (datetime.datetime.utcnow() -
self.fullTimeLog[transID]
) < datetime.timedelta(
seconds=self.fullUpdatePeriod):
timeStamp = self.timeLog[transID]
if self.dateKey:
inputDataQuery[self.dateKey] = (
timeStamp - datetime.timedelta(seconds=10)
).strftime("%Y-%m-%d %H:%M:%S")
else:
self.log.error(
"DateKey was not set in the CS, cannot use the RefreshOnly"
)
else:
self.fullTimeLog[
transID] = datetime.datetime.utcnow()
self.timeLog[transID] = datetime.datetime.utcnow()
if transID not in self.fullTimeLog:
self.fullTimeLog[transID] = datetime.datetime.utcnow()
# Perform the query to the metadata catalog
self.log.verbose("Using input data query for transformation",
"%d: %s" % (transID, str(inputDataQuery)))
start = time.time()
result = self.metadataClient.findFilesByMetadata(inputDataQuery)
rtime = time.time() - start
self.log.verbose("Metadata catalog query time",
": %.2f seconds." % (rtime))
if not result["OK"]:
self.log.error(
"InputDataAgent.execute: Failed to get response from the metadata catalog",
result["Message"])
continue
lfnList = result["Value"]
# Check if the number of files has changed since the last cycle
nlfns = len(lfnList)
self.log.info(
"files returned for transformation from the metadata catalog: ",
"%d -> %d" % (int(transID), nlfns))
if nlfns == self.fileLog.get(transID):
self.log.verbose(
"No new files in metadata catalog since last check")
self.fileLog[transID] = nlfns
# Add any new files to the transformation
addedLfns = []
if lfnList:
self.log.verbose("Processing lfns for transformation:",
"%d -> %d" % (transID, len(lfnList)))
# Add the files to the transformation
self.log.verbose("Adding lfns for transformation:",
"%d -> %d" % (transID, len(lfnList)))
result = self.transClient.addFilesToTransformation(
transID, sorted(lfnList))
if not result["OK"]:
self.log.warn(
"InputDataAgent.execute: failed to add lfns to transformation",
result["Message"])
self.fileLog[transID] = 0
else:
if result["Value"]["Failed"]:
for lfn, error in res["Value"]["Failed"].items():
self.log.warn(
"InputDataAgent.execute: Failed to add to transformation:",
"%s: %s" % (lfn, error))
if result["Value"]["Successful"]:
for lfn, status in result["Value"]["Successful"].items(
):
if status == "Added":
addedLfns.append(lfn)
self.log.info(
"InputDataAgent.execute: Added files to transformation",
"(%d)" % len(addedLfns))
return S_OK()
Script.showHelp(exitCode=1)
# get arguments
inputFileName = args[1]
lfns = []
if os.path.exists(inputFileName):
inputFile = open(inputFileName, 'r')
string = inputFile.read()
inputFile.close()
lfns.extend([lfn.strip() for lfn in string.splitlines()])
else:
lfns.append(inputFileName)
tc = TransformationClient()
res = tc.addFilesToTransformation(args[0], lfns) # Files added here
if not res['OK']:
DIRAC.gLogger.error(res['Message'])
DIRAC.exit(2)
successfullyAdded = 0
alreadyPresent = 0
for lfn, message in res['Value']['Successful'].items():
if message == 'Added':
successfullyAdded += 1
elif message == 'Present':
alreadyPresent += 1
if successfullyAdded > 0:
DIRAC.gLogger.notice("Successfully added %d files" % successfullyAdded)
class InputDataAgent( AgentModule ):
def __init__( self, *args, **kwargs ):
''' c'tor
'''
AgentModule.__init__( self, *args, **kwargs )
self.fileLog = {}
self.timeLog = {}
self.fullTimeLog = {}
self.pollingTime = self.am_getOption( 'PollingTime', 120 )
self.fullUpdatePeriod = self.am_getOption( 'FullUpdatePeriod', 86400 )
self.refreshonly = self.am_getOption( 'RefreshOnly', False )
self.dateKey = self.am_getOption( 'DateKey', None )
self.transClient = TransformationClient()
self.metadataClient = FileCatalogClient()
self.transformationTypes = None
#############################################################################
def initialize( self ):
''' Make the necessary initializations
'''
gMonitor.registerActivity( "Iteration", "Agent Loops", AGENT_NAME, "Loops/min", gMonitor.OP_SUM )
agentTSTypes = self.am_getOption( 'TransformationTypes', [] )
if agentTSTypes:
self.transformationTypes = sorted( agentTSTypes )
else:
dataProc = Operations().getValue( 'Transformations/DataProcessing', ['MCSimulation', 'Merge'] )
dataManip = Operations().getValue( 'Transformations/DataManipulation', ['Replication', 'Removal'] )
self.transformationTypes = sorted( dataProc + dataManip )
extendables = Operations().getValue( 'Transformations/ExtendableTransfTypes', [])
if extendables:
for extendable in extendables:
if extendable in self.transformationTypes:
self.transformationTypes.remove(extendable)
#This is because the Extendables do not use this Agent (have no Input data query)
return S_OK()
##############################################################################
def execute( self ):
''' Main execution method
'''
gMonitor.addMark( 'Iteration', 1 )
# Get all the transformations
result = self.transClient.getTransformations( {'Status' : 'Active',
'Type' : self.transformationTypes } )
if not result['OK']:
gLogger.error( "InputDataAgent.execute: Failed to get transformations.", result['Message'] )
return S_OK()
# Process each transformation
for transDict in result['Value']:
transID = long( transDict['TransformationID'] )
res = self.transClient.getTransformationInputDataQuery( transID )
if not res['OK']:
if res['Message'] == 'No InputDataQuery found for transformation':
gLogger.info( "InputDataAgent.execute: No input data query found for transformation %d" % transID )
else:
gLogger.error( "InputDataAgent.execute: Failed to get input data query for %d" % transID, res['Message'] )
continue
inputDataQuery = res['Value']
if self.refreshonly:
# Determine the correct time stamp to use for this transformation
if self.timeLog.has_key( transID ):
if self.fullTimeLog.has_key( transID ):
# If it is more than a day since the last reduced query, make a full query just in case
if ( datetime.datetime.utcnow() - self.fullTimeLog[transID] ) < datetime.timedelta( seconds = self.fullUpdatePeriod ):
timeStamp = self.timeLog[transID]
if self.dateKey:
inputDataQuery[self.dateKey] = ( timeStamp - datetime.timedelta( seconds = 10 ) ).strftime( '%Y-%m-%d %H:%M:%S' )
else:
gLogger.error( "DateKey was not set in the CS, cannot use the RefreshOnly" )
else:
self.fullTimeLog[transID] = datetime.datetime.utcnow()
self.timeLog[transID] = datetime.datetime.utcnow()
if not self.fullTimeLog.has_key( transID ):
self.fullTimeLog[transID] = datetime.datetime.utcnow()
# Perform the query to the metadata catalog
gLogger.verbose( "Using input data query for transformation %d: %s" % ( transID, str( inputDataQuery ) ) )
start = time.time()
result = self.metadataClient.findFilesByMetadata( inputDataQuery )
rtime = time.time() - start
gLogger.verbose( "Metadata catalog query time: %.2f seconds." % ( rtime ) )
if not result['OK']:
gLogger.error( "InputDataAgent.execute: Failed to get response from the metadata catalog", result['Message'] )
continue
lfnList = result['Value']
# Check if the number of files has changed since the last cycle
nlfns = len( lfnList )
gLogger.info( "%d files returned for transformation %d from the metadata catalog" % ( nlfns, int( transID ) ) )
if self.fileLog.has_key( transID ):
if nlfns == self.fileLog[transID]:
gLogger.verbose( 'No new files in metadata catalog since last check' )
self.fileLog[transID] = nlfns
# Add any new files to the transformation
addedLfns = []
if lfnList:
gLogger.verbose( 'Processing %d lfns for transformation %d' % ( len( lfnList ), transID ) )
# Add the files to the transformation
gLogger.verbose( 'Adding %d lfns for transformation %d' % ( len( lfnList ), transID ) )
result = self.transClient.addFilesToTransformation( transID, sorted( lfnList ) )
if not result['OK']:
gLogger.warn( "InputDataAgent.execute: failed to add lfns to transformation", result['Message'] )
self.fileLog[transID] = 0
else:
if result['Value']['Failed']:
for lfn, error in res['Value']['Failed'].items():
gLogger.warn( "InputDataAgent.execute: Failed to add %s to transformation" % lfn, error )
if result['Value']['Successful']:
for lfn, status in result['Value']['Successful'].items():
if status == 'Added':
addedLfns.append( lfn )
gLogger.info( "InputDataAgent.execute: Added %d files to transformation" % len( addedLfns ) )
return S_OK()
dl_count_step2 = [None] * N_TH_STEP_2
dl_count_step3 = [None] * N_TH_STEP_3
f = []
for r in range(0, N_TH_STEP_1):
min_tmp = min[r]
if ((min_tmp + N_TH_STEP_3) >= n_file[r]):
# Si verifica se (e quanti) File bisogna creare per aggiungere un Task per una specifica Trasformazione
diff_file = ((min_tmp + N_TH_STEP_3) - n_file[r]) + 1
# Si crea il numero giusto di File guardando se il numero di file da creare e' positivo
if (diff_file > 0):
for i in range(0, diff_file):
min_tmp += 1
#lfns_tmp = "/aa/lfn." + str(min_tmp) + "." + str(r) + ".txt"
lfns_tmp = "/aa/lfn." + str(uuid_str) + "." + str(
i) + ".txt"
res = transClient.addFilesToTransformation(
transID[r], [lfns_tmp])
if res['OK'] == True:
print "**** Creato un File per creare un nuovo Task (per il thread ", r + 1, ") ****"
lfns[r].append(lfns_tmp)
else:
print "\033[1;31m#### ERROR: File per aggiungere i Task non creati ####\033[1;m"
print res['Message']
min_tmp = min[r]
f_tmp = []
for i in range(0, N_TH_STEP_3):
min_tmp = min_tmp + 1
f_tmp.append([lfns[r][min_tmp]])
f.append(f_tmp)
for i in range(1, N_TH_STEP_2 + 1):
r = random.randint(0, N_TH_STEP_1 - 1)
t.setTransformationName(transferName) # Must be unique
t.setTransformationGroup("Transfer")
t.setType("Transfer-JUNO")
#t.setPlugin("Standard") # Not needed. The default is 'Standard'
t.setDescription("Test Data Transfer")
t.setLongDescription( "Long description of Data Transfer" ) # Mandatory
t.setGroupSize(3) # Here you specify how many files should be grouped within he same request, e.g. 100
transBody = [ ( "ReplicateAndRegister", { "SourceSE": fromSE, "TargetSE": toSE }) ]
t.setBody ( transBody ) # Mandatory
result = t.addTransformation() # Transformation is created here
if not result['OK']:
gLogger.error('Can not add transformation: %s' % result['Message'])
exit(2)
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
result = tc.addFilesToTransformation(transID['Value'], infileList) # Files are added here
if not result['OK']:
gLogger.error('Can not add files to transformation: %s' % result['Message'])
exit(2)
result = tc.setTransformationParameter( transID['Value'], 'Status', 'Flush' )
if not result['OK']:
gLogger.error('Can not flush transformation: %s' % result['Message'])
exit(2)
times_s3 = [None]*N_TH_STEP_3
dl_count_step2 = [None]*N_TH_STEP_2
dl_count_step3 = [None]*N_TH_STEP_3
f = []
for r in range(0,N_TH_STEP_1):
min_tmp = min[r]
if ((min_tmp + N_TH_STEP_3) >= n_file[r]):
# Si verifica se (e quanti) File bisogna creare per aggiungere un Task per una specifica Trasformazione
diff_file = ((min_tmp + N_TH_STEP_3) - n_file[r]) +1
# Si crea il numero giusto di File guardando se il numero di file da creare e' positivo
if (diff_file > 0):
for i in range(0,diff_file):
min_tmp += 1
#lfns_tmp = "/aa/lfn." + str(min_tmp) + "." + str(r) + ".txt"
lfns_tmp = "/aa/lfn." + str(uuid_str) + "." + str(i) + ".txt"
res = transClient.addFilesToTransformation( transID[r], [lfns_tmp] )
if res['OK']==True:
print "**** Creato un File per creare un nuovo Task (per il thread ",r+1,") ****"
lfns[r].append(lfns_tmp)
else:
print "\033[1;31m#### ERROR: File per aggiungere i Task non creati ####\033[1;m"
print res['Message']
min_tmp = min[r]
f_tmp = []
for i in range(0,N_TH_STEP_3):
min_tmp = min_tmp + 1
f_tmp.append([lfns[r][min_tmp]])
f.append(f_tmp)
for i in range(1, N_TH_STEP_2 + 1):
r = random.randint(0, N_TH_STEP_1 - 1)
class InputDataAgent(AgentModule):
#############################################################################
def initialize(self):
""" Make the necessary initializations """
self.fileLog = {}
self.timeLog = {}
self.fullTimeLog = {}
self.pollingTime = self.am_getOption('PollingTime',120)
self.fullUpdatePeriod = self.am_getOption('FullUpdatePeriod',86400)
gMonitor.registerActivity("Iteration","Agent Loops",AGENT_NAME,"Loops/min",gMonitor.OP_SUM)
self.transClient = TransformationClient('TransformationDB')
self.metadataClient = FileCatalogClient()
return S_OK()
##############################################################################
def execute(self):
""" Main execution method
"""
gMonitor.addMark('Iteration',1)
# Get all the transformations
result = self.transClient.getTransformations(condDict={'Status':'Active'})
activeTransforms = []
if not result['OK']:
gLogger.error("InputDataAgent.execute: Failed to get transformations.", result['Message'])
return S_OK()
# Process each transformation
for transDict in result['Value']:
transID = long(transDict['TransformationID'])
res = self.transClient.getTransformationInputDataQuery(transID)
if not res['OK']:
if res['Message'] == 'No InputDataQuery found for transformation':
gLogger.info("InputDataAgent.execute: No input data query found for transformation %d" % transID)
else:
gLogger.error("InputDataAgent.execute: Failed to get input data query for %d" % transID, res['Message'])
continue
inputDataQuery = res['Value']
# Determine the correct time stamp to use for this transformation
if self.timeLog.has_key(transID):
if self.fullTimeLog.has_key(transID):
# If it is more than a day since the last reduced query, make a full query just in case
if (datetime.datetime.utcnow() - self.fullTimeLog[transID]) < datetime.timedelta(seconds=self.fullUpdatePeriod):
timeStamp = self.timeLog[transID]
inputDataQuery['StartDate'] = (timeStamp - datetime.timedelta(seconds=10)).strftime('%Y-%m-%d %H:%M:%S')
else:
self.fullTimeLog[transID] = datetime.datetime.utcnow()
self.timeLog[transID] = datetime.datetime.utcnow()
if not self.fullTimeLog.has_key(transID):
self.fullTimeLog[transID] = datetime.datetime.utcnow()
# Perform the query to the metadata catalog
gLogger.verbose("Using input data query for transformation %d: %s" % (transID,str(inputDataQuery)))
start = time.time()
result = self.metadataClient.findFilesByMetadata(inputDataQuery)
rtime = time.time()-start
gLogger.verbose("Metadata catalog query time: %.2f seconds." % (rtime))
if not result['OK']:
gLogger.error("InputDataAgent.execute: Failed to get response from the metadata catalog", result['Message'])
continue
lfnList = result['Value']
# Check if the number of files has changed since the last cycle
nlfns = len(lfnList)
gLogger.info("%d files returned for transformation %d from the metadata catalog" % (nlfns,int(transID)) )
if self.fileLog.has_key(transID):
if nlfns == self.fileLog[transID]:
gLogger.verbose('No new files in metadata catalog since last check')
self.fileLog[transID] = nlfns
# Add any new files to the transformation
addedLfns = []
if lfnList:
gLogger.verbose('Processing %d lfns for transformation %d' % (len(lfnList),transID) )
# Add the files to the transformation
gLogger.verbose('Adding %d lfns for transformation %d' % (len(lfnList),transID) )
result = self.transClient.addFilesToTransformation(transID,sortList(lfnList))
if not result['OK']:
gLogger.warn("InputDataAgent.execute: failed to add lfns to transformation", result['Message'])
self.fileLog[transID] = 0
else:
if result['Value']['Failed']:
for lfn,error in res['Value']['Failed'].items():
gLogger.warn("InputDataAgent.execute: Failed to add %s to transformation" % lfn,error)
if result['Value']['Successful']:
for lfn,status in result['Value']['Successful'].items():
if status == 'Added':
addedLfns.append(lfn)
gLogger.info("InputDataAgent.execute: Added %d files to transformation" % len(addedLfns))
return S_OK()