def DataReplicationByQueryTSExample(args=None):
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
t = Transformation()
tc = TransformationClient()
t.setTransformationName("DM_ReplicationByQuery1") # This must vary
#t.setTransformationGroup("Group1")
t.setType("Replication")
t.setSourceSE(['CYF-STORM-Disk', 'DESY-ZN-Disk'
]) # A list of SE where at least 1 SE is the valid one
t.setTargetSE(['CEA-Disk'])
t.setDescription("data Replication")
t.setLongDescription("data Replication") #mandatory
t.setGroupSize(1)
t.setPlugin("Broadcast")
t.addTransformation() #transformation is created here
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
tc.createTransformationInputDataQuery(
transID['Value'], {
'particle': 'gamma',
'prodName': 'Config_test300113',
'outputType': 'Data',
'simtelArrayProdVersion': 'prod-2_21122012_simtel',
'runNumSeries': '0'
}) # Add files to Transformation based on Catalog Query
def DataRemovalByQueryTSExample(args=None):
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
t = Transformation()
tc = TransformationClient()
t.setTransformationName("DM_RemovalQuery1") # Must be unique
#t.setTransformationGroup("Group1")
t.setType("Removal")
t.setPlugin("Standard") # Not needed. The default is 'Standard'
t.setDescription("corsika Removal")
t.setLongDescription("corsika Removal") # Mandatory
t.setGroupSize(
2
) # Here you specify how many files should be grouped within the same request, e.g. 100
t.setBody("Removal;RemoveFile"
) # Mandatory (the default is a ReplicateAndRegister operation)
t.addTransformation() # Transformation is created here
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
tc.createTransformationInputDataQuery(transID['Value'], {
'particle': 'proton',
'prodName': 'ConfigTestTS9',
'outputType': 'Data'
}) # Add files to Transformation based on Catalog Query
def ReproTSExample( args = None ) :
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
from CTADIRAC.Interfaces.API.ReproTSJob import ReproTSJob
if (len(args) != 2):
Script.gLogger.notice('Wrong number of arguments')
Script.showHelp()
infile = args[0]
f = open(infile,'r')
infileList = []
for line in f:
infile = line.strip()
if line!="\n":
infileList.append(infile)
if args[1] not in ['STD','NSBX3','SCMST','SCSST','4MSST','ASTRI','NORTH']:
Script.gLogger.notice('reprocessing configuration incorrect:',args[1])
Script.showHelp()
simtelArrayConfig = args[1]
j = ReproTSJob()
j.setVersion('prod-2_15122013')
j.setParameters(['fileCatalog.cfg','-S',simtelArrayConfig])
j.setInputSandbox( ['LFN:/vo.cta.in2p3.fr/MC/PROD2/CFG_rev6956/prod2_cfg.tar.gz'])
j.setOutputSandbox( ['*.log','applicationLog.txt'])
j.setCPUTime(200000)
t = Transformation( )
tc = TransformationClient( )
t.setTransformationName("Reprotest1") # This must vary
#t.setTransformationGroup("Group1")
t.setType("DataReprocessing")
t.setDescription("simtel repro example")
t.setLongDescription( "simtel reprocessing" ) #mandatory
t.setBody ( j.workflow.toXML() )
t.addTransformation() #transformation is created here
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
tc.addFilesToTransformation(transID['Value'],infileList) # Files added here
def submitTS():
########################################
# Modify here with your dirac username
owner = 'user02'
########################################
########################################
# Transformation definition
########################################
t = Transformation( )
t.setTransformationName( owner+'_step4' )
t.setType("Removal")
t.setDescription("Remove intermediate mandelbrot images")
t.setLongDescription( "Remove intermediate mandelbrot images" )
# set the request to be executed
t.setBody ( "Removal;RemoveFile" ) # Mandatory (the default is a ReplicateAndRegister operation)
# define input data by metadata query
inputMetaquery = {"application":"mandelbrot","image_format":"ascii","owner":owner}
t.setInputMetaQuery(inputMetaquery)
########################################
# Transformation submission
########################################
res = t.addTransformation()
if not res['OK']:
print(res['Message'])
DIRAC.exit( -1 )
t.setStatus("Active")
t.setAgentType("Automatic")
return res
def submitTS(job):
""" Create a transformation executing the job workflow """
### Temporary fix to initialize JOB_ID #######
job.workflow.addParameter(
Parameter("JOB_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
job.workflow.addParameter(
Parameter("PRODUCTION_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
job.setType('MCSimulation') ## Used for the JobType plugin
t = Transformation()
# t.setTransformationName("Prod3Exemple") # This must be unique. If not set it's asked in the prompt
t.setType("MCSimulation")
t.setDescription("MC Prod3 BaseLine HE test")
t.setLongDescription("corsika-simtel production") # mandatory
t.setBody(job.workflow.toXML())
res = t.addTransformation() # Transformation is created here
if not res['OK']:
print(res['Message'])
DIRAC.exit(-1)
t.setStatus("Active")
t.setAgentType("Automatic")
return res
def submit_trans(job, trans_name, input_meta_query, group_size):
""" Create a transformation executing the job workflow
"""
DIRAC.gLogger.notice('submit_trans : %s' % trans_name)
# Initialize JOB_ID
job.workflow.addParameter(
Parameter("JOB_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
trans = Transformation()
trans.setTransformationName(trans_name) # this must be unique
trans.setType("DataReprocessing")
trans.setDescription("Prod5 EventDisplay TS")
trans.setLongDescription("Prod5 EventDisplay processing") # mandatory
trans.setBody(job.workflow.toXML())
trans.setGroupSize(group_size)
trans.setInputMetaQuery(input_meta_query)
result = trans.addTransformation() # transformation is created here
if not result['OK']:
return result
trans.setStatus("Active")
trans.setAgentType("Automatic")
trans_id = trans.getTransformationID()
return trans_id
def submit_trans(job, trans_name, mqJson, group_size, with_file_mask=True):
""" Create a transformation executing the job workflow
"""
DIRAC.gLogger.notice('submit_trans : %s' % trans_name)
# Initialize JOB_ID
job.workflow.addParameter(
Parameter("JOB_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
trans = Transformation()
trans.setTransformationName(trans_name) # this must be unique
trans.setType("DataReprocessing")
trans.setDescription("Prod3 DL1 Data Handler TS")
trans.setLongDescription("Prod3 DL1 Data Handler conversion") # mandatory
trans.setBody(job.workflow.toXML())
trans.setGroupSize(group_size)
if with_file_mask:
trans.setFileMask(mqJson) # catalog query is defined here
result = trans.addTransformation() # transformation is created here
if not result['OK']:
return result
trans.setStatus("Active")
trans.setAgentType("Automatic")
trans_id = trans.getTransformationID()
return trans_id
def submit_trans(job, infileList, trans_name, group_size):
""" Create a transformation executing the job workflow
"""
DIRAC.gLogger.notice('submit_trans : %s' % trans_name)
# Initialize JOB_ID
job.workflow.addParameter(
Parameter("JOB_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
trans = Transformation()
trans.setTransformationName(trans_name) # this must be unique
trans.setType("DataReprocessing")
trans.setDescription("Simtel TS example")
trans.setLongDescription("Simtel tel_sim") # mandatory
trans.setBody(job.workflow.toXML())
trans.setGroupSize(group_size)
res = trans.addTransformation() # transformation is created here
if not res['OK']:
return res
trans.setStatus("Active")
trans.setAgentType("Automatic")
# add 10*group_size files to transformation (to have the first 10 jobs)
trans_id = trans.getTransformationID()
trans_client = TransformationClient()
res = trans_client.addFilesToTransformation(trans_id['Value'],
infileList[:10 * group_size])
return res
def submit_TS_no_MQ(job, infileList):
""" Create a transformation executing the job workflow
"""
t = Transformation()
tc = TransformationClient()
t.setType("DataReprocessing")
t.setDescription("Runs ImageExtractor analysis for array HB9 SCT")
t.setLongDescription(
"merge_simtel, ImageExtractor DL0->DL1 conversion for HB9 SCT")
t.setGroupSize(5)
t.setBody(job.workflow.toXML())
res = t.addTransformation() # Transformation is created here
if not res['OK']:
Script.gLogger.error(res['Message'])
DIRAC.exit(-1)
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
Script.gLogger.notice('Adding %s files to transformation' %
len(infileList))
tc.addFilesToTransformation(transID['Value'], infileList)
return res
def submit_trans(job, trans_name):
""" Create a transformation executing the job workflow """
DIRAC.gLogger.notice('submit_trans : %s' % trans_name)
# Initialize JOB_ID
job.workflow.addParameter(
Parameter("JOB_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
job.workflow.addParameter(
Parameter("PRODUCTION_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
job.setType('MCSimulation') ## Used for the JobType plugin
trans = Transformation()
trans.setTransformationName(trans_name)
trans.setType("MCSimulation")
trans.setDescription("MC Prod3 BaseLine Corsika7 test")
trans.setLongDescription("corsika-simtel production") # mandatory
trans.setBody(job.workflow.toXML())
result = trans.addTransformation() # transformation is created here
if not result['OK']:
return result
trans.setStatus("Active")
trans.setAgentType("Automatic")
trans_id = trans.getTransformationID()
return trans_id
def OldSubmitTS(job, infileList):
""" Create a transformation executing the job workflow """
t = Transformation()
tc = TransformationClient()
t.setType("DataReprocessing")
t.setDescription("Runs EvnDisp analysis for array HB9 SCT")
t.setLongDescription(
"merge_simtel, evndisp converter and evndisp analysis for HB9 SCT"
) # mandatory
t.setGroupSize(5)
t.setBody(job.workflow.toXML())
res = t.addTransformation() # Transformation is created here
if not res['OK']:
print res['Message']
DIRAC.exit(-1)
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
print('Adding %s files to transformation' % len(infileList))
tc.addFilesToTransformation(transID['Value'],
infileList) # Files added here
return res
def submit_trans(job, input_meta_query, group_size):
""" Create a transformation executing the job workflow """
#DIRAC.gLogger.notice('submit_trans : %s' % trans_name)
# Initialize JOB_ID
job.workflow.addParameter(
Parameter("JOB_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
t = Transformation()
t.setType("DataReprocessing")
t.setDescription("EvnDisplay MQ example")
t.setLongDescription("EvnDisplay calib_imgreco") # mandatory
t.setBody(job.workflow.toXML())
t.setGroupSize(group_size)
t.setInputMetaQuery(input_meta_query)
res = t.addTransformation() # transformation is created here
if not res['OK']:
return res
#t.setStatus("Active")
t.setAgentType("Automatic")
trans_id = t.getTransformationID()
return trans_id
def DataReplicaRemovalTSExample(args=None):
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
if (len(args) != 1):
Script.gLogger.notice('Wrong number of arguments')
Script.showHelp()
infile = args[0]
f = open(infile, 'r')
infileList = []
for line in f:
infile = line.strip()
if line != "\n":
infileList.append(infile)
t = Transformation()
tc = TransformationClient()
t.setType("Removal")
t.setPlugin("Broadcast") # Mandatory for ReplicaRemoval
t.setSourceSE(['CC-IN2P3-Tape'
]) # A list of SE where at least 1 SE is the valid one
t.setTargetSE([
'CEA-Disk', 'LAPP-Disk', 'DESY-ZN-Disk', 'CC-IN2P3-Disk',
'CYF-STORM-Disk'
]) # A list of SE where at least 1 SE is the valid one
t.setDescription("AarProd2 ReplicaRemoval")
t.setLongDescription("AarProd2 ReplicaRemoval") # Mandatory
t.setGroupSize(
100
) # Here you specify how many files should be grouped within the same request, e.g. 100
t.setBody("Removal;RemoveReplica"
) # Mandatory (the default is a ReplicateAndRegister operation)
t.addTransformation() # Transformation is created here
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
tc.addFilesToTransformation(transID['Value'],
infileList) # Files are added here
def DataReplicationTSExample(args=None):
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
if (len(args) != 1):
Script.gLogger.notice('Wrong number of arguments')
Script.showHelp()
infile = args[0]
f = open(infile, 'r')
infileList = []
for line in f:
infile = line.strip()
if line != "\n":
infileList.append(infile)
t = Transformation()
tc = TransformationClient()
t.setTransformationName("DM_Replication") # This must be unique
#t.setTransformationGroup("Group1")
t.setType("Replication")
t.setSourceSE(['CYF-STORM-Disk', 'DESY-ZN-Disk'
]) # A list of SE where at least 1 SE is the valid one
t.setTargetSE(['CEA-Disk'])
t.setDescription("corsika Replication")
t.setLongDescription("corsika Replication") # Mandatory
t.setGroupSize(
1
) # Here you specify how many files should be grouped within the same request, e.g. 100
t.setPlugin("Broadcast") # Mandatory for replication
t.addTransformation() # Transformation is created here
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
tc.addFilesToTransformation(transID['Value'],
infileList) # Files are added here
def DataRemovalTSExample(args=None):
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
if (len(args) != 1):
Script.gLogger.notice('Wrong number of arguments')
Script.showHelp()
infile = args[0]
f = open(infile, 'r')
infileList = []
for line in f:
infile = line.strip()
if line != "\n":
infileList.append(infile)
t = Transformation()
tc = TransformationClient()
t.setTransformationName("DM_Removal26") # Must be unique
#t.setTransformationGroup("Group1")
t.setType("Removal")
t.setPlugin("Standard") # Not needed. The default is 'Standard'
t.setDescription("corsika Removal")
t.setLongDescription("corsika Removal") # Mandatory
t.setGroupSize(
1
) # Here you specify how many files should be grouped within the same request, e.g. 100
t.setBody("Removal;RemoveFile"
) # Mandatory (the default is a ReplicateAndRegister operation)
t.addTransformation() # Transformation is created here
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
tc.addFilesToTransformation(transID['Value'],
infileList) # Files are added here
def submit_trans(job, transName, mqJson, group_size):
""" Create a transformation executing the job workflow
Input files are given through as an existing data set
"""
DIRAC.gLogger.notice('submit_trans : %s' % transName)
# Initialize JOB_ID
job.workflow.addParameter(Parameter("JOB_ID", "000000", "string", "", "",
True, False, "Temporary fix"))
trans = Transformation()
trans.setTransformationName(transName) # this must be unique
trans.setType("DataReprocessing")
trans.setDescription("SimpleCtapipe example")
trans.setLongDescription("ctapipe classify, reconstruct and merge: calib_imgreco") # mandatory
trans.setBody(job.workflow.toXML())
trans.setGroupSize(group_size)
trans.setFileMask(mqJson) # catalog query is defined here
trans.addTransformation() # transformation is created here
trans.setStatus("Active")
trans.setAgentType("Automatic")
return trans
def submitTS(job, transName, mqJson):
""" Create a transformation executing the job workflow """
DIRAC.gLogger.notice('submitTS')
# Initialize JOB_ID
job.workflow.addParameter(
Parameter("JOB_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
t = Transformation()
t.setTransformationName(transName) # this must be unique
t.setType("DataReprocessing")
t.setDescription("EvnDisplay MQ example")
t.setLongDescription("EvnDisplay calib_imgreco") # mandatory
t.setBody(job.workflow.toXML())
t.setGroupSize(5)
t.setFileMask(mqJson) # catalog query is defined here
t.addTransformation() # transformation is created here
t.setStatus("Active")
t.setAgentType("Automatic")
return
def _createTransformation(self,transName,plugin='Standard'):
oTrans = Transformation()
res = oTrans.setTransformationName(transName)
self.assert_(res['OK'])
description = 'Test transforamtion description'
res = oTrans.setDescription(description)
longDescription = 'Test transformation long description'
res = oTrans.setLongDescription(longDescription)
self.assert_(res['OK'])
res = oTrans.setType('MCSimulation')
self.assert_(res['OK'])
res = oTrans.setPlugin(plugin)
self.assert_(res['OK'])
res = oTrans.addTransformation()
self.assert_(res['OK'])
self.transID = res['Value']
return res
def _createTransformation(self, transName, plugin='Standard'):
oTrans = Transformation()
res = oTrans.setTransformationName(transName)
self.assert_(res['OK'])
description = 'Test transforamtion description'
res = oTrans.setDescription(description)
longDescription = 'Test transformation long description'
res = oTrans.setLongDescription(longDescription)
self.assert_(res['OK'])
res = oTrans.setType('MCSimulation')
self.assert_(res['OK'])
res = oTrans.setPlugin(plugin)
self.assert_(res['OK'])
res = oTrans.addTransformation()
self.assert_(res['OK'])
self.transID = res['Value']
return res
def createTransformation(self):
########################################
# Transformation definition
########################################
t = Transformation()
t.setTransformationName(self.__transName)
t.setType(self.__transType)
t.setDescription(self.__description)
t.setLongDescription(self.__description)
t.setGroupSize(self.__groupSize)
if self.__transGroup:
t.setTransformationGroup(self.__transGroup)
t.setPlugin(self.__plugin)
t.setTargetSE(self.__targetSE)
transBody = []
t.setBody(transBody)
########################################
# Transformation submission
########################################
res = t.addTransformation()
if not res['OK']:
raise Exception('Add transformation error: {0}'.format(
res['Message']))
t.setStatus("Active")
t.setAgentType("Automatic")
currtrans = t.getTransformationID()['Value']
if self.__inputMeta:
client = TransformationClient()
res = client.createTransformationMetaQuery(currtrans,
self.__inputMeta,
'Input')
if not res['OK']:
raise Exception(
'Create transformation query error: {0}'.format(
res['Message']))
return str(currtrans)
def _createReplication(targetSE, sourceSE, prodID, datatype, extraname=''):
"""Creates the replication transformation based on the given parameters"""
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
metadata = {"Datatype": datatype, "ProdID": prodID}
trans = Transformation()
transName = 'replicate_%s_%s' % (str(prodID), ",".join(targetSE))
if extraname:
transName += "_%s" % extraname
trans.setTransformationName(transName)
description = 'Replicate files for prodID %s to %s' % (str(prodID),
",".join(targetSE))
trans.setDescription(description)
trans.setLongDescription(description)
trans.setType('Replication')
trans.setGroup('Replication')
trans.setPlugin('Broadcast')
res = trans.setSourceSE(sourceSE)
if not res['OK']:
exit(1)
res = trans.setTargetSE(targetSE)
if not res['OK']:
exit(1)
res = trans.addTransformation()
if not res['OK']:
gLogger.error(res['Message'])
exit(1)
gLogger.verbose(res)
trans.setStatus('Active')
trans.setAgentType('Automatic')
currtrans = trans.getTransformationID()['Value']
client = TransformationClient()
res = client.createTransformationInputDataQuery(currtrans, metadata)
if res['OK']:
gLogger.always("Successfully created replication transformation")
return S_OK()
else:
gLogger.error("Failure during replication creation", res['Message'])
return S_ERROR("Failed to create transformation")
def createTransformation(self):
########################################
# Transformation definition
########################################
t = Transformation()
t.setTransformationName(self.__stepName)
t.setType(self.__transType)
t.setDescription(self.__description)
t.setLongDescription(self.__description)
if self.__isGen:
t.setMaxNumberOfTasks(self.__maxNumberOfTasks)
else:
t.setGroupSize(1)
if self.__transGroup:
t.setTransformationGroup(self.__transGroup)
# set the job workflow to the transformation
t.setBody(self.__job.workflow.toXML())
########################################
# Transformation submission
########################################
res = t.addTransformation()
if not res['OK']:
raise Exception('Add transformation error: {0}'.format(
res['Message']))
t.setStatus("Active")
t.setAgentType("Automatic")
currtrans = t.getTransformationID()['Value']
if self.__inputMeta and not self.__isGen:
client = TransformationClient()
res = client.createTransformationInputDataQuery(
currtrans, self.__inputMeta)
if not res['OK']:
raise Exception(
'Create transformation query error: {0}'.format(
res['Message']))
return str(currtrans)
def _createReplication( targetSE, sourceSE, prodID, datatype, extraname=''):
"""Creates the replication transformation based on the given parameters"""
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
metadata = {"Datatype":datatype, "ProdID":prodID}
trans = Transformation()
transName = 'replicate_%s_%s' % ( str(prodID), ",".join(targetSE) )
if extraname:
transName += "_%s" % extraname
trans.setTransformationName( transName )
description = 'Replicate files for prodID %s to %s' % ( str(prodID), ",".join(targetSE) )
trans.setDescription( description )
trans.setLongDescription( description )
trans.setType( 'Replication' )
trans.setPlugin( 'Broadcast' )
res = trans.setSourceSE( sourceSE )
if not res['OK']:
exit(1)
res = trans.setTargetSE( targetSE )
if not res['OK']:
exit(1)
res = trans.addTransformation()
if not res['OK']:
gLogger.error(res['Message'])
exit(1)
gLogger.verbose(res)
trans.setStatus( 'Active' )
trans.setAgentType( 'Automatic' )
currtrans = trans.getTransformationID()['Value']
client = TransformationClient()
res = client.createTransformationInputDataQuery( currtrans, metadata )
if res['OK']:
gLogger.always("Successfully created replication transformation")
return S_OK()
else:
gLogger.error("Failure during replication creation", res['Message'])
return S_ERROR("Failed to create transformation")
def submit_trans(job, trans_name):
""" Create a transformation executing the job workflow
"""
DIRAC.gLogger.notice('submit_trans : %s' % trans_name)
# Initialize JOB_ID
job.workflow.addParameter(Parameter("JOB_ID", "000000", "string", "", "",
True, False, "Temporary fix"))
trans = Transformation()
trans.setTransformationName(trans_name) # this must be unique
trans.setType("MCSimulation")
trans.setDescription("Prod5 MC Pipe NSB TS")
trans.setLongDescription("Prod5 simulation pipeline") # mandatory
trans.setBody(job.workflow.toXML())
result = trans.addTransformation() # transformation is created here
if not result['OK']:
return result
trans.setStatus("Active")
trans.setAgentType("Automatic")
trans_id = trans.getTransformationID()
return trans_id
def DataReplicationByQueryTSExample( args = None ) :
t = Transformation( )
t.setType("Replication")
t.setSourceSE(['CYF-STORM-Disk','DESY-ZN-Disk']) # A list of SE where at least 1 SE is the valid one
t.setTargetSE(['CC-IN2P3-Disk'])
t.setDescription("data Replication")
t.setLongDescription( "data Replication" ) #mandatory
t.setGroupSize(10)
t.setPlugin("Broadcast")
inputQuery = {'MCCampaign':'PROD4','site':'Paranal','calibimgreco_prog':'evndisp','calibimgreco_prog_version':'prod4_d20181110','particle':'gamma','array_layout':'Baseline-SST-only','configuration_id':4,'phiP':180.0,'thetaP':20.0,'outputType':'Data','data_level': 1}
t.setInputMetaQuery(inputQuery)
res = t.addTransformation() #transformation is created here
if not res['OK']:
DIRAC.gLogger.error(res['Message'])
DIRAC.exit(-1)
def submitTS( job, infileList ):
""" Create a transformation executing the job workflow """
t = Transformation()
tc = TransformationClient()
t.setType( "DataReprocessing" )
t.setDescription( "Chimp example" )
t.setLongDescription( "Chimp analysis" ) # mandatory
t.setGroupSize(1)
t.setBody ( job.workflow.toXML() )
res = t.addTransformation() # Transformation is created here
if not res['OK']:
print res['Message']
DIRAC.exit( -1 )
t.setStatus( "Active" )
t.setAgentType( "Automatic" )
transID = t.getTransformationID()
tc.addFilesToTransformation( transID['Value'], infileList ) # Files added here
return res
def submit_trans(job, trans_name, infileList, group_size):
""" Create a transformation executing the job workflow
"""
DIRAC.gLogger.notice('submit_trans : %s' % trans_name)
# Initialize JOB_ID
job.workflow.addParameter(
Parameter("JOB_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
trans = Transformation()
trans.setTransformationName(trans_name) # this must be unique
trans.setType("DataReprocessing")
trans.setDescription("SimpleCtapipe example")
trans.setLongDescription(
"ctapipe classify, reconstruct and merge: calib_imgreco") # mandatory
trans.setBody(job.workflow.toXML())
trans.setGroupSize(group_size)
res = trans.addTransformation() # transformation is created here
if not res['OK']:
DIRAC.gLogger.error('Error creating transformation: ', res['Message'])
return res
transID = res['Value']
# Activate the transformation
trans.setStatus("Active")
trans.setAgentType("Automatic")
# Add files to the transformation
tc = TransformationClient()
res = tc.addFilesToTransformation(transID, infileList)
if not res['OK']:
DIRAC.gLogger.error('Error adding files to the transformation: ',
res['Message'])
return res
def submitTS( job, infileList ):
""" Create a transformation executing the job workflow """
t = Transformation()
tc = TransformationClient()
t.setType( "SimtelMerging" )
t.setDescription( "Runs merge_simtel for array 3HB89" )
t.setLongDescription( "Merging array 3HB89 analysis" ) # mandatory
t.setGroupSize(5)
t.setBody ( job.workflow.toXML() )
res = t.addTransformation() # Transformation is created here
if not res['OK']:
print res['Message']
DIRAC.exit( -1 )
t.setStatus( "Active" )
t.setAgentType( "Automatic" )
transID = t.getTransformationID()
print('Adding %s files to transformation'%len(infileList))
tc.addFilesToTransformation( transID['Value'], infileList ) # Files added here
return res
def DataRemovalByQueryTSExample(args=None):
t = Transformation()
t.setType("Removal")
t.setPlugin("Standard") # Not needed. The default is 'Standard'
t.setDescription("corsika Removal")
t.setLongDescription("corsika Removal") # Mandatory
t.setGroupSize(
100
) # Here you specify how many files should be grouped within the same request, e.g. 100
t.setBody("Removal;RemoveFile"
) # Mandatory (the default is a ReplicateAndRegister operation)
inputQuery = {"prodName": "ConfigSAC_24062013"}
t.setInputMetaQuery(inputQuery)
res = t.addTransformation() # Transformation is created here
if not res['OK']:
DIRAC.gLogger.error(res['Message'])
DIRAC.exit(-1)
def createDataTransformation(
flavour,
targetSE,
sourceSE,
metaKey,
metaValue,
extraData=None,
extraname="",
groupSize=1,
plugin="Broadcast",
tGroup=None,
tBody=None,
enable=False,
):
"""Creates the replication transformation based on the given parameters.
:param str flavour: Flavour of replication to create: Replication or Moving
:param targetSE: Destination for files
:type targetSE: python:list or str
:param sourceSE: Origin of files
:type sourceSE: python:list or str
:param int metaKey: Meta key to identify input files
:param int metaValue: Meta value to identify input files
:param dict metaData: Additional meta data to use to identify input files
:param str extraname: addition to the transformation name, only needed if the same transformation was already created
:param int groupSize: number of files per transformation taks
:param str plugin: plugin to use
:param str tGroup: transformation group to set
:param tBody: transformation body to set
:param bool enable: if true submit the transformation, otherwise dry run
:returns: S_OK (with the transformation object, if successfully added), S_ERROR
"""
metadata = {metaKey: metaValue}
if isinstance(extraData, dict):
metadata.update(extraData)
gLogger.debug("Using %r for metadata search" % metadata)
if isinstance(targetSE, six.string_types):
targetSE = [targetSE]
if isinstance(sourceSE, (list, tuple)):
sourceSE = "%s" % (",".join(sourceSE))
gLogger.debug("Using plugin: %r" % plugin)
if flavour not in ("Replication", "Moving"):
return S_ERROR("Unsupported flavour %s" % flavour)
transVerb = {"Replication": "Replicate", "Moving": "Move"}[flavour]
transGroup = {
"Replication": "Replication",
"Moving": "Moving"
}[flavour] if not tGroup else tGroup
trans = Transformation()
transName = "%s_%s_%s" % (transVerb, str(metaValue), ",".join(targetSE))
if extraname:
transName += "_%s" % extraname
trans.setTransformationName(transName)
description = "%s files for %s %s to %s" % (
transVerb, metaKey, str(metaValue), ",".join(targetSE))
trans.setDescription(description[:255])
trans.setLongDescription(description)
trans.setType("Replication")
trans.setTransformationGroup(transGroup)
trans.setGroupSize(groupSize)
trans.setPlugin(plugin)
transBody = ({
"Moving": [
("ReplicateAndRegister", {
"SourceSE": sourceSE,
"TargetSE": targetSE
}),
("RemoveReplica", {
"TargetSE": sourceSE
}),
],
"Replication":
"", # empty body
}[flavour] if tBody is None else tBody)
trans.setBody(transBody)
trans.setInputMetaQuery(metadata)
if sourceSE:
res = trans.setSourceSE(sourceSE)
if not res["OK"]:
return S_ERROR("SourceSE not valid: %s" % res["Message"])
res = trans.setTargetSE(targetSE)
if not res["OK"]:
return S_ERROR("TargetSE not valid: %s" % res["Message"])
if not enable:
gLogger.always("Dry run, not creating transformation")
return S_OK()
res = trans.addTransformation()
if not res["OK"]:
return res
gLogger.verbose(res)
trans.setStatus("Active")
trans.setAgentType("Automatic")
gLogger.always("Successfully created replication transformation")
return S_OK(trans)
def submitTS():
########################################
# Modify here with your dirac username
owner = 'user02'
########################################
########################################
# Job description
########################################
job = Job()
job.setName('build mandelbrot')
job.setOutputSandbox( ['*log'] )
job.setType('DataReprocessing')
## define the job workflow in 3 steps
# job step1: setup software
job.setExecutable('git clone https://github.com/bregeon/mandel4ts.git')
# job step2: run mandelbrot build image
job.setExecutable('./mandel4ts/build_merged_img.py')
outputPath = os.path.join('/vo.france-grilles.fr/user',owner[0],owner,'mandelbrot/images/final')
outputPattern = 'merged_image.bmp'
outputSE = 'DIRAC-USER'
outputMetadata = json.dumps( {"application":"mandelbrot","image_format":"bmp", "image_width":7680, "image_height":4200, "owner":owner} )
# job step3: upload data and set metadata
job.setExecutable( './mandel4ts/dirac-add-files.py', arguments = "%s '%s' %s '%s'" % (outputPath, outputPattern, outputSE, outputMetadata ) )
# job step4: mark input files as done with the FailoverRequest (and a few other things)
job.setExecutable('/bin/ls -l', modulesList=['Script', 'FailoverRequest'])
########################################
# Transformation definition
########################################
t = Transformation()
t.setTransformationName( owner+'_step3' )
t.setType( "DataReprocessing" )
t.setDescription( "Merge mandelbrot images production" )
t.setLongDescription( "Merge mandelbrot images production" )
t.setGroupSize( 3 ) # group input files
# set the job workflow to the transformation
t.setBody ( job.workflow.toXML() )
# define input data by metadata query
inputMetaquery = {"application":"mandelbrot","image_format":"ascii", "image_width":7680, "image_height":1400, "owner":owner}
t.setInputMetaQuery(inputMetaquery)
########################################
# Transformation submission
########################################
res = t.addTransformation()
if not res['OK']:
print(res['Message'])
DIRAC.exit( -1 )
t.setStatus( "Active" )
t.setAgentType( "Automatic" )
return res
def createDataTransformation(flavour, targetSE, sourceSE,
metaKey, metaValue,
extraData=None, extraname='',
groupSize=1,
plugin='Broadcast',
tGroup=None,
tBody=None,
enable=False,
):
"""Creates the replication transformation based on the given parameters.
:param str flavour: Flavour of replication to create: Replication or Moving
:param targetSE: Destination for files
:type targetSE: python:list or str
:param str sourceSE: Origin of files.
:param int metaKey: Meta key to identify input files
:param int metaValue: Meta value to identify input files
:param dict metaData: Additional meta data to use to identify input files
:param str extraname: addition to the transformation name, only needed if the same transformation was already created
:param int groupSize: number of files per transformation taks
:param str plugin: plugin to use
:param str tGroup: transformation group to set
:param tBody: transformation body to set
:param bool enable: if true submit the transformation, otherwise dry run
:returns: S_OK (with the transformation object, if successfully added), S_ERROR
"""
metadata = {metaKey: metaValue}
if isinstance(extraData, dict):
metadata.update(extraData)
gLogger.debug("Using %r for metadata search" % metadata)
if isinstance(targetSE, basestring):
targetSE = [targetSE]
if flavour not in ('Replication', 'Moving'):
return S_ERROR('Unsupported flavour %s' % flavour)
transVerb = {'Replication': 'Replicate', 'Moving': 'Move'}[flavour]
transGroup = {'Replication': 'Replication', 'Moving': 'Moving'}[flavour] if not tGroup else tGroup
trans = Transformation()
transName = '%s_%s_%s' % (transVerb, str(metaValue), ",".join(targetSE))
if extraname:
transName += "_%s" % extraname
trans.setTransformationName(transName)
description = '%s files for %s %s to %s' % (transVerb, metaKey, str(metaValue), ",".join(targetSE))
trans.setDescription(description)
trans.setLongDescription(description)
trans.setType('Replication')
trans.setTransformationGroup(transGroup)
trans.setGroupSize(groupSize)
trans.setPlugin(plugin)
transBody = {'Moving': [("ReplicateAndRegister", {"SourceSE": sourceSE, "TargetSE": targetSE}),
("RemoveReplica", {"TargetSE": sourceSE})],
'Replication': '', # empty body
}[flavour] if tBody is None else tBody
trans.setBody(transBody)
if sourceSE:
res = trans.setSourceSE(sourceSE)
if not res['OK']:
return S_ERROR("SourceSE not valid: %s" % res['Message'])
res = trans.setTargetSE(targetSE)
if not res['OK']:
return S_ERROR("TargetSE not valid: %s" % res['Message'])
if not enable:
gLogger.always("Dry run, not creating transformation")
return S_OK()
res = trans.addTransformation()
if not res['OK']:
return res
gLogger.verbose(res)
trans.setStatus('Active')
trans.setAgentType('Automatic')
currtrans = trans.getTransformationID()['Value']
client = TransformationClient()
res = client.createTransformationInputDataQuery(currtrans, metadata)
if not res['OK']:
return res
gLogger.always("Successfully created replication transformation")
return S_OK(trans)
) # Here you specify how many files should be grouped within he same request, e.g. 100
transBody = ''
t.setBody(transBody)
if sourceSE:
res = t.setSourceSE(sourceSE)
if not res['OK']:
gLogger.error("SourceSE not valid: %s" % res['Message'])
exit(1)
res = t.setTargetSE(targetSE)
if not res['OK']:
gLogger.error("TargetSE not valid: %s" % res['Message'])
exit(1)
result = t.addTransformation() # Transformation is created here
if not result['OK']:
gLogger.error('Can not add transformation: %s' % result['Message'])
exit(2)
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
result = tc.createTransformationInputDataQuery(transID['Value'], query)
if not result['OK']:
gLogger.error('Can not create query to transformation: %s' %
result['Message'])
exit(2)
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
from DIRAC.Interfaces.API.Job import Job
j = Job()
tc = TransformationClient()
t = Transformation()
t.setTransformationName("Un exemple") #<- unique
t.setTransformationGroup("Un groupe") #<- for monitoring
t.setType("MCSimulation")#type must be among known types
t.setDescription("Ceci est un exemple")
t.setLongDescription("C'est un bel exemple")
t.setBody(j.workflow._toXML())
t.setGroupSize(1)
t.setPlugin("Standard")
t.addTransformation() #<-- transformation is created here
t.setStatus("Active") #<-- make it start
t.setAgentType("Automatic") #<-- should be by default
transfid = t.getTransformationID()['Value'] #<- unique
tc.createTransformationInputDataQuery(transfid,
{'meta1':val1,"meta2":{">":34}})
gLogger.error("Failed to add metadata fields", res['Message'])
exit(-1)
# Set directory meta data
timestamp = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime())
MDdict1 = {'particle': 'gamma', 'timestamp': timestamp}
res = fc.setMetadata(directory, MDdict1)
if not res['OK']:
gLogger.error("Failed to set metadata", res['Message'])
exit(-1)
# Set the transformation meta data filter
MDdict1b = {'particle': 'gamma', 'timestamp': timestamp}
mqJson1b = json.dumps(MDdict1b)
res = transformation.setFileMask(mqJson1b)
if not res['OK']:
gLogger.error("Failed to set FileMask", res['Message'])
exit(-1)
# Create the transformation
result = transformation.addTransformation()
if not result['OK']:
print result
exit(1)
transID = result['Value']
with open('TransformationID', 'w') as fd:
fd.write(str(transID))
print "Created %s, stored in file 'TransformationID'" % transID
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
# Define transformation steps for the replication of the output data
if replicateFiles and meta:
Trans = Transformation()
Trans.setTransformationName( 'replicate_%s_%s_%s_%s' % ( process, energy, polarisation, meta['Datatype'] ) )
description = 'Replicate %s %s %s %s to' % ( process, energy, polarisation, meta['Datatype'] )
for replicaSRM in replicaSRMs:
description += ' %s,' % ( replicaSRM )
description.rstrip( ',' )
Trans.setDescription( description )
Trans.setLongDescription( description )
Trans.setType( 'Replication' )
Trans.setPlugin( 'Broadcast' )
Trans.setSourceSE( outputSRM )
Trans.setTargetSE( replicaSRMs )
res = Trans.addTransformation()
if not res['OK']:
print res
sys.exit(0)
print res
Trans.setStatus( 'Active' )
Trans.setAgentType( 'Automatic' )
currtrans = Trans.getTransformationID()['Value']
client = TransformationClient()
res = client.createTransformationInputDataQuery( currtrans, meta )
print res['OK']
