def test_getTransformations(self):
""" Testing the selection of transformations from the database
getTransformations
This will select all the transformations associated to this test suite and remove them.
"""
oTrans = Transformation()
res = oTrans.getTransformations()
self.assert_(res['OK'])
parameters = [
'TransformationID', 'TransformationName', 'Description',
'LongDescription', 'CreationDate', 'LastUpdate', 'AuthorDN',
'AuthorGroup', 'Type', 'Plugin', 'AgentType', 'Status', 'FileMask',
'TransformationGroup', 'GroupSize', 'InheritedFrom', 'Body',
'MaxNumberOfTasks', 'EventsPerTask'
]
self.assertEqual(sortList(res['ParameterNames']), sortList(parameters))
self.assertEqual(sortList(res['Value'][0].keys()),
sortList(parameters))
self.assertEqual(len(res['Value']), len(res['Records']))
ignore = self.transID
for transDict in res['Value']:
name = transDict['TransformationName']
if re.search('TestClientTransformation', name):
transID = transDict['TransformationID']
if transID != ignore:
oTrans = Transformation(transID)
res = oTrans.deleteTransformation()
self.assert_(res['OK'])
self.transID = ignore
def submit_TS_no_MQ(job, infileList):
""" Create a transformation executing the job workflow
"""
t = Transformation()
tc = TransformationClient()
t.setType("DataReprocessing")
t.setDescription("Runs ImageExtractor analysis for array HB9 SCT")
t.setLongDescription(
"merge_simtel, ImageExtractor DL0->DL1 conversion for HB9 SCT")
t.setGroupSize(5)
t.setBody(job.workflow.toXML())
res = t.addTransformation() # Transformation is created here
if not res['OK']:
Script.gLogger.error(res['Message'])
DIRAC.exit(-1)
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
Script.gLogger.notice('Adding %s files to transformation' %
len(infileList))
tc.addFilesToTransformation(transID['Value'], infileList)
return res
def DataReplicationByQueryTSExample(args=None):
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
t = Transformation()
tc = TransformationClient()
t.setTransformationName("DM_ReplicationByQuery1") # This must vary
#t.setTransformationGroup("Group1")
t.setType("Replication")
t.setSourceSE(['CYF-STORM-Disk', 'DESY-ZN-Disk'
]) # A list of SE where at least 1 SE is the valid one
t.setTargetSE(['CEA-Disk'])
t.setDescription("data Replication")
t.setLongDescription("data Replication") #mandatory
t.setGroupSize(1)
t.setPlugin("Broadcast")
t.addTransformation() #transformation is created here
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
tc.createTransformationInputDataQuery(
transID['Value'], {
'particle': 'gamma',
'prodName': 'Config_test300113',
'outputType': 'Data',
'simtelArrayProdVersion': 'prod-2_21122012_simtel',
'runNumSeries': '0'
}) # Add files to Transformation based on Catalog Query
def submit_trans(job, trans_name):
""" Create a transformation executing the job workflow """
DIRAC.gLogger.notice('submit_trans : %s' % trans_name)
# Initialize JOB_ID
job.workflow.addParameter(
Parameter("JOB_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
job.workflow.addParameter(
Parameter("PRODUCTION_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
job.setType('MCSimulation') ## Used for the JobType plugin
trans = Transformation()
trans.setTransformationName(trans_name)
trans.setType("MCSimulation")
trans.setDescription("MC Prod3 BaseLine Corsika7 test")
trans.setLongDescription("corsika-simtel production") # mandatory
trans.setBody(job.workflow.toXML())
result = trans.addTransformation() # transformation is created here
if not result['OK']:
return result
trans.setStatus("Active")
trans.setAgentType("Automatic")
trans_id = trans.getTransformationID()
return trans_id
def setUp( self ):
self.mockTransClient = Mock()
self.mockTransClient.setTaskStatusAndWmsID.return_value = {'OK':True}
self.WMSClientMock = Mock()
self.jobMonitoringClient = Mock()
self.mockReqClient = Mock()
self.jobMock = Mock()
self.jobMock2 = Mock()
mockWF = Mock()
mockPar = Mock()
mockWF.findParameter.return_value = mockPar
mockPar.getValue.return_value = 'MySite'
self.jobMock2.workflow = mockWF
self.jobMock2.setDestination.return_value = {'OK':True}
self.jobMock.workflow.return_value = ''
self.jobMock.return_value = self.jobMock2
self.taskBase = TaskBase( transClient = self.mockTransClient )
self.wfTasks = WorkflowTasks( transClient = self.mockTransClient,
submissionClient = self.WMSClientMock,
jobMonitoringClient = self.jobMonitoringClient,
outputDataModule = "mock",
jobClass = self.jobMock )
self.requestTasks = RequestTasks( transClient = self.mockTransClient,
requestClient = self.mockReqClient
)
self.tc = TransformationClient()
self.transformation = Transformation()
self.maxDiff = None
def DataRemovalByQueryTSExample(args=None):
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
t = Transformation()
tc = TransformationClient()
t.setTransformationName("DM_RemovalQuery1") # Must be unique
#t.setTransformationGroup("Group1")
t.setType("Removal")
t.setPlugin("Standard") # Not needed. The default is 'Standard'
t.setDescription("corsika Removal")
t.setLongDescription("corsika Removal") # Mandatory
t.setGroupSize(
2
) # Here you specify how many files should be grouped within the same request, e.g. 100
t.setBody("Removal;RemoveFile"
) # Mandatory (the default is a ReplicateAndRegister operation)
t.addTransformation() # Transformation is created here
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
tc.createTransformationInputDataQuery(transID['Value'], {
'particle': 'proton',
'prodName': 'ConfigTestTS9',
'outputType': 'Data'
}) # Add files to Transformation based on Catalog Query
def do_getall(self, args):
"""Get transformation details
usage: getall [Status] [Status]
"""
oTrans = Transformation()
oTrans.getTransformations(transStatus=args.split(), printOutput=True)
def submit_trans(job, trans_name, mqJson, group_size, with_file_mask=True):
""" Create a transformation executing the job workflow
"""
DIRAC.gLogger.notice('submit_trans : %s' % trans_name)
# Initialize JOB_ID
job.workflow.addParameter(
Parameter("JOB_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
trans = Transformation()
trans.setTransformationName(trans_name) # this must be unique
trans.setType("DataReprocessing")
trans.setDescription("Prod3 DL1 Data Handler TS")
trans.setLongDescription("Prod3 DL1 Data Handler conversion") # mandatory
trans.setBody(job.workflow.toXML())
trans.setGroupSize(group_size)
if with_file_mask:
trans.setFileMask(mqJson) # catalog query is defined here
result = trans.addTransformation() # transformation is created here
if not result['OK']:
return result
trans.setStatus("Active")
trans.setAgentType("Automatic")
trans_id = trans.getTransformationID()
return trans_id
def OldSubmitTS(job, infileList):
""" Create a transformation executing the job workflow """
t = Transformation()
tc = TransformationClient()
t.setType("DataReprocessing")
t.setDescription("Runs EvnDisp analysis for array HB9 SCT")
t.setLongDescription(
"merge_simtel, evndisp converter and evndisp analysis for HB9 SCT"
) # mandatory
t.setGroupSize(5)
t.setBody(job.workflow.toXML())
res = t.addTransformation() # Transformation is created here
if not res['OK']:
print res['Message']
DIRAC.exit(-1)
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
print('Adding %s files to transformation' % len(infileList))
tc.addFilesToTransformation(transID['Value'],
infileList) # Files added here
return res
def submit_trans(job, infileList, trans_name, group_size):
""" Create a transformation executing the job workflow
"""
DIRAC.gLogger.notice('submit_trans : %s' % trans_name)
# Initialize JOB_ID
job.workflow.addParameter(
Parameter("JOB_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
trans = Transformation()
trans.setTransformationName(trans_name) # this must be unique
trans.setType("DataReprocessing")
trans.setDescription("Simtel TS example")
trans.setLongDescription("Simtel tel_sim") # mandatory
trans.setBody(job.workflow.toXML())
trans.setGroupSize(group_size)
res = trans.addTransformation() # transformation is created here
if not res['OK']:
return res
trans.setStatus("Active")
trans.setAgentType("Automatic")
# add 10*group_size files to transformation (to have the first 10 jobs)
trans_id = trans.getTransformationID()
trans_client = TransformationClient()
res = trans_client.addFilesToTransformation(trans_id['Value'],
infileList[:10 * group_size])
return res
def submitTS():
########################################
# Modify here with your dirac username
owner = 'user02'
########################################
########################################
# Transformation definition
########################################
t = Transformation( )
t.setTransformationName( owner+'_step4' )
t.setType("Removal")
t.setDescription("Remove intermediate mandelbrot images")
t.setLongDescription( "Remove intermediate mandelbrot images" )
# set the request to be executed
t.setBody ( "Removal;RemoveFile" ) # Mandatory (the default is a ReplicateAndRegister operation)
# define input data by metadata query
inputMetaquery = {"application":"mandelbrot","image_format":"ascii","owner":owner}
t.setInputMetaQuery(inputMetaquery)
########################################
# Transformation submission
########################################
res = t.addTransformation()
if not res['OK']:
print(res['Message'])
DIRAC.exit( -1 )
t.setStatus("Active")
t.setAgentType("Automatic")
return res
def submit_trans(job, trans_name, input_meta_query, group_size):
""" Create a transformation executing the job workflow
"""
DIRAC.gLogger.notice('submit_trans : %s' % trans_name)
# Initialize JOB_ID
job.workflow.addParameter(
Parameter("JOB_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
trans = Transformation()
trans.setTransformationName(trans_name) # this must be unique
trans.setType("DataReprocessing")
trans.setDescription("Prod5 EventDisplay TS")
trans.setLongDescription("Prod5 EventDisplay processing") # mandatory
trans.setBody(job.workflow.toXML())
trans.setGroupSize(group_size)
trans.setInputMetaQuery(input_meta_query)
result = trans.addTransformation() # transformation is created here
if not result['OK']:
return result
trans.setStatus("Active")
trans.setAgentType("Automatic")
trans_id = trans.getTransformationID()
return trans_id
def submit_trans(job, input_meta_query, group_size):
""" Create a transformation executing the job workflow """
#DIRAC.gLogger.notice('submit_trans : %s' % trans_name)
# Initialize JOB_ID
job.workflow.addParameter(
Parameter("JOB_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
t = Transformation()
t.setType("DataReprocessing")
t.setDescription("EvnDisplay MQ example")
t.setLongDescription("EvnDisplay calib_imgreco") # mandatory
t.setBody(job.workflow.toXML())
t.setGroupSize(group_size)
t.setInputMetaQuery(input_meta_query)
res = t.addTransformation() # transformation is created here
if not res['OK']:
return res
#t.setStatus("Active")
t.setAgentType("Automatic")
trans_id = t.getTransformationID()
return trans_id
def __init__(self):
"""Create a string containing restructured text documentation for all the special tranformation parameters
:param str module: full path to the module
:param bool replace: wether to replace the full docstring or not
if you replace it completely add the automodule commands etc.!
"""
self.module = MODULE
self.replace = False
self.doc_string = ''
# Fill the docstring addition with what we want to add
self.doc_string += textwrap.dedent("""
Transformation Parameters
-------------------------
Any parameter with ``ParameterName`` can be set for a transformation with a call
to ``setParameterName(parameterValue)``.
The following parameters have a special meaning
""")
from DIRAC.TransformationSystem.Client.Transformation import Transformation
trans = Transformation()
for paramName in sorted(trans.paramTypes):
self.doc_string += '\n``%s``:\n Default value: %r' % (
paramName, trans.paramValues[paramName])
def submitTS(job):
""" Create a transformation executing the job workflow """
### Temporary fix to initialize JOB_ID #######
job.workflow.addParameter(
Parameter("JOB_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
job.workflow.addParameter(
Parameter("PRODUCTION_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
job.setType('MCSimulation') ## Used for the JobType plugin
t = Transformation()
# t.setTransformationName("Prod3Exemple") # This must be unique. If not set it's asked in the prompt
t.setType("MCSimulation")
t.setDescription("MC Prod3 BaseLine HE test")
t.setLongDescription("corsika-simtel production") # mandatory
t.setBody(job.workflow.toXML())
res = t.addTransformation() # Transformation is created here
if not res['OK']:
print(res['Message'])
DIRAC.exit(-1)
t.setStatus("Active")
t.setAgentType("Automatic")
return res
def do_getall( self, args ):
"""Get transformation details
usage: getall [Status] [Status]
"""
oTrans = Transformation()
oTrans.setServer( self.serverURL )
oTrans.getTransformations( transStatus = string.split( args ), printOutput = True )
def do_getall(self, args):
'''Get transformation details
usage: getall [Status] [Status]
'''
oTrans = Transformation()
oTrans.setServer(self.serverURL)
oTrans.getTransformations(transStatus=args.split(), printOutput=True)
def test_AddFilesGetFilesSetFileStatus(self):
""" Testing adding, getting and setting file status.
addFilesToTransformation()
getTransformationFiles()
getTransformationStats()
setFileStatusForTransformation()
addTaskForTransformation()
Test adding and files to transformation.
Test selecting the files for the transformation.
Test getting the status count of the transformation files.
Test setting the file status for transformation.
Test creating a task for the added files and ensure the status is updated correctly.
"""
oTrans = Transformation(self.transID)
lfns = ['/test/lfn/file1', '/test/lfn/file2']
res = oTrans.addFilesToTransformation(lfns)
self.assert_(res['OK'])
res = oTrans.getTransformationFiles()
self.assert_(res['OK'])
self.assertEqual(sortList(lfns), res['LFNs'])
self.assertEqual(len(lfns), len(res['Records']))
self.assertEqual(len(lfns), len(res['Value']))
fileParams = sortList([
'LFN', 'TransformationID', 'FileID', 'Status', 'TaskID',
'TargetSE', 'UsedSE', 'ErrorCount', 'LastUpdate', 'InsertedTime'
])
self.assertEqual(fileParams, sortList(res['ParameterNames']))
self.assertEqual(res['Records'][0][0], lfns[0])
self.assertEqual(res['Value'][0]['LFN'], lfns[0])
self.assertEqual(res['Records'][0][1], self.transID)
self.assertEqual(res['Value'][0]['TransformationID'], self.transID)
self.assertEqual(res['Records'][0][3], 'Unused')
self.assertEqual(res['Value'][0]['Status'], 'Unused')
res = oTrans.getTransformationStats()
self.assert_(res['OK'])
self.assertEqual(res['Value']['Total'], 2)
self.assertEqual(res['Value']['Unused'], 2)
res = oTrans.setFileStatusForTransformation('Processed', [lfns[0]])
self.assert_(res['OK'])
res = oTrans.getTransformationStats()
self.assert_(res['OK'])
self.assertEqual(res['Value']['Total'], 2)
self.assertEqual(res['Value']['Unused'], 1)
self.assertEqual(res['Value']['Processed'], 1)
res = oTrans.setFileStatusForTransformation('Unused', [lfns[0]])
self.assert_(res['OK'])
self.assert_(res['Value']['Failed'].has_key(lfns[0]))
self.assertEqual(res['Value']['Failed'][lfns[0]],
'Can not change Processed status')
res = oTrans.addTaskForTransformation(lfns=[lfns[1]], se='Test')
self.assert_(res['OK'])
res = oTrans.getTransformationStats()
self.assert_(res['OK'])
self.assertEqual(res['Value']['Total'], 2)
self.assertEqual(res['Value']['Assigned'], 1)
self.assertEqual(res['Value']['Processed'], 1)
def ReproTSExample( args = None ) :
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
from CTADIRAC.Interfaces.API.ReproTSJob import ReproTSJob
if (len(args) != 2):
Script.gLogger.notice('Wrong number of arguments')
Script.showHelp()
infile = args[0]
f = open(infile,'r')
infileList = []
for line in f:
infile = line.strip()
if line!="\n":
infileList.append(infile)
if args[1] not in ['STD','NSBX3','SCMST','SCSST','4MSST','ASTRI','NORTH']:
Script.gLogger.notice('reprocessing configuration incorrect:',args[1])
Script.showHelp()
simtelArrayConfig = args[1]
j = ReproTSJob()
j.setVersion('prod-2_15122013')
j.setParameters(['fileCatalog.cfg','-S',simtelArrayConfig])
j.setInputSandbox( ['LFN:/vo.cta.in2p3.fr/MC/PROD2/CFG_rev6956/prod2_cfg.tar.gz'])
j.setOutputSandbox( ['*.log','applicationLog.txt'])
j.setCPUTime(200000)
t = Transformation( )
tc = TransformationClient( )
t.setTransformationName("Reprotest1") # This must vary
#t.setTransformationGroup("Group1")
t.setType("DataReprocessing")
t.setDescription("simtel repro example")
t.setLongDescription( "simtel reprocessing" ) #mandatory
t.setBody ( j.workflow.toXML() )
t.addTransformation() #transformation is created here
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
tc.addFilesToTransformation(transID['Value'],infileList) # Files added here
def do_getAllByUser(self, args):
"""Get all transformations created by a given user
The first argument is the authorDN or username. The authorDN
is preferred: it need to be inside quotes because contains
white spaces. Only authorDN should be quoted.
When the username is provided instead,
the authorDN is retrieved from the uploaded proxy,
so that the retrieved transformations are those created by
the user who uploaded that proxy: that user could be different
that the username provided to the function.
usage: getAllByUser authorDN or username [Status] [Status]
"""
oTrans = Transformation()
argss = args.split()
username = ""
author = ""
status = []
if not len(argss) > 0:
print self.do_getAllByUser.__doc__
return
# if the user didnt quoted the authorDN ends
if '=' in argss[0] and argss[0][0] not in ["'", '"']:
print "AuthorDN need to be quoted (just quote that argument)"
return
if argss[0][0] in ["'", '"']: # authorDN given
author = argss[0]
status_idx = 1
for arg in argss[1:]:
author += ' ' + arg
status_idx += 1
if arg[-1] in ["'", '"']:
break
# At this point we should have something like 'author'
if not author[0] in ["'", '"'] or not author[-1] in ["'", '"']:
print "AuthorDN need to be quoted (just quote that argument)"
return
else:
author = author[1:-1] # throw away the quotes
# the rest are the requested status
status = argss[status_idx:]
else: # username given
username = argss[0]
status = argss[1:]
oTrans.getTransformationsByUser(authorDN=author,
userName=username,
transStatus=status,
printOutput=True)
def test_SetGetReset(self):
""" Testing of the set, get and reset methods.
set*()
get*()
setTargetSE()
setSourceSE()
getTargetSE()
getSourceSE()
reset()
Ensures that after a reset all parameters are returned to their defaults
"""
oTrans = Transformation()
res = oTrans.getParameters()
self.assert_(res['OK'])
defaultParams = res['Value'].copy()
for parameterName, defaultValue in res['Value'].items():
if type(defaultValue) in types.StringTypes:
testValue = "'TestValue'"
else:
testValue = '99999'
execString = "res = oTrans.set%s(%s)" % (parameterName, testValue)
exec(execString)
self.assert_(res['OK'])
execString = "res = oTrans.get%s()" % (parameterName)
exec(execString)
self.assert_(res['OK'])
self.assertEqual(res['Value'], eval(testValue))
# Test that SEs delivered as a space or comma seperated string are resolved...
stringSEs = 'CERN-USER, CNAF-USER GRIDKA-USER,IN2P3-USER'
listSEs = stringSEs.replace(',', ' ').split()
res = oTrans.setTargetSE(stringSEs)
self.assert_(res['OK'])
res = oTrans.getTargetSE()
self.assert_(res['OK'])
self.assertEqual(res['Value'], listSEs)
# ...but that lists are correctly handled also
res = oTrans.setSourceSE(listSEs)
self.assert_(res['OK'])
res = oTrans.getSourceSE()
self.assert_(res['OK'])
self.assertEqual(res['Value'], listSEs)
res = oTrans.reset()
self.assert_(res['OK'])
res = oTrans.getParameters()
self.assert_(res['OK'])
for parameterName, resetValue in res['Value'].items():
self.assertEqual(resetValue, defaultParams[parameterName])
self.assertRaises(AttributeError, oTrans.getTargetSE)
self.assertRaises(AttributeError, oTrans.getSourceSE)
def test_DeleteTransformation(self):
""" Testing of the deletion of a transformation.
addTransformation()
deleteTransformation()
Tests that a transformation can be removed.
Tests that retrieving a non existant transformation raises an AttributeError.
"""
oTrans = Transformation(self.transID)
res = oTrans.deleteTransformation()
self.assert_(res['OK'])
self.assertRaises(AttributeError, Transformation, self.transID)
self.transID = 0
def setUp(self):
self.mockTransClient = mock.MagicMock()
self.mockTransClient.setTaskStatusAndWmsID.return_value = {'OK': True}
self.mockReqClient = mock.MagicMock()
self.taskBase = TaskBase(transClient=self.mockTransClient)
self.pu = PluginUtilities(transClient=self.mockTransClient)
self.requestTasks = RequestTasks(transClient=self.mockTransClient,
requestClient=self.mockReqClient,
requestValidator=reqValFake)
self.transformation = Transformation()
self.maxDiff = None
def do_summaryTransformations(self, args):
"""Show the summary for a list of Transformations
Fields starting with 'F' ('J') refers to files (jobs).
Proc. stand for processed.
Usage: summaryTransformations <ProdID> [<ProdID> ...]
"""
argss = args.split()
if not len(argss) > 0:
print self.do_summaryTransformations.__doc__
return
transid = argss
oTrans = Transformation()
oTrans.getSummaryTransformations(transID=transid)
def DataReplicaRemovalTSExample(args=None):
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
if (len(args) != 1):
Script.gLogger.notice('Wrong number of arguments')
Script.showHelp()
infile = args[0]
f = open(infile, 'r')
infileList = []
for line in f:
infile = line.strip()
if line != "\n":
infileList.append(infile)
t = Transformation()
tc = TransformationClient()
t.setType("Removal")
t.setPlugin("Broadcast") # Mandatory for ReplicaRemoval
t.setSourceSE(['CC-IN2P3-Tape'
]) # A list of SE where at least 1 SE is the valid one
t.setTargetSE([
'CEA-Disk', 'LAPP-Disk', 'DESY-ZN-Disk', 'CC-IN2P3-Disk',
'CYF-STORM-Disk'
]) # A list of SE where at least 1 SE is the valid one
t.setDescription("AarProd2 ReplicaRemoval")
t.setLongDescription("AarProd2 ReplicaRemoval") # Mandatory
t.setGroupSize(
100
) # Here you specify how many files should be grouped within the same request, e.g. 100
t.setBody("Removal;RemoveReplica"
) # Mandatory (the default is a ReplicateAndRegister operation)
t.addTransformation() # Transformation is created here
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
tc.addFilesToTransformation(transID['Value'],
infileList) # Files are added here
def _createTransformation(self, transName, plugin='Standard'):
oTrans = Transformation()
res = oTrans.setTransformationName(transName)
self.assert_(res['OK'])
description = 'Test transforamtion description'
res = oTrans.setDescription(description)
longDescription = 'Test transformation long description'
res = oTrans.setLongDescription(longDescription)
self.assert_(res['OK'])
res = oTrans.setType('MCSimulation')
self.assert_(res['OK'])
res = oTrans.setPlugin(plugin)
self.assert_(res['OK'])
res = oTrans.addTransformation()
self.assert_(res['OK'])
self.transID = res['Value']
return res
def test_getTransformationLogging(self):
""" Testing the obtaining of transformation logging information
getTransformationLogging()
"""
oTrans = Transformation(self.transID)
res = oTrans.setStatus('Active')
self.assert_(res['OK'])
res = oTrans.extendTransformation(100)
self.assert_(res['OK'])
res = oTrans.setStatus('Completed')
self.assert_(res['OK'])
res = oTrans.cleanTransformation()
self.assert_(res['OK'])
res = oTrans.getTransformationLogging()
self.assert_(res['OK'])
self.assertEqual(len(res['Value']), 6)
def createTransformation(self):
########################################
# Transformation definition
########################################
t = Transformation()
t.setTransformationName(self.__transName)
t.setType(self.__transType)
t.setDescription(self.__description)
t.setLongDescription(self.__description)
t.setGroupSize(self.__groupSize)
if self.__transGroup:
t.setTransformationGroup(self.__transGroup)
t.setPlugin(self.__plugin)
t.setTargetSE(self.__targetSE)
transBody = []
t.setBody(transBody)
########################################
# Transformation submission
########################################
res = t.addTransformation()
if not res['OK']:
raise Exception('Add transformation error: {0}'.format(
res['Message']))
t.setStatus("Active")
t.setAgentType("Automatic")
currtrans = t.getTransformationID()['Value']
if self.__inputMeta:
client = TransformationClient()
res = client.createTransformationMetaQuery(currtrans,
self.__inputMeta,
'Input')
if not res['OK']:
raise Exception(
'Create transformation query error: {0}'.format(
res['Message']))
return str(currtrans)
def _createReplication(targetSE, sourceSE, prodID, datatype, extraname=''):
"""Creates the replication transformation based on the given parameters"""
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
metadata = {"Datatype": datatype, "ProdID": prodID}
trans = Transformation()
transName = 'replicate_%s_%s' % (str(prodID), ",".join(targetSE))
if extraname:
transName += "_%s" % extraname
trans.setTransformationName(transName)
description = 'Replicate files for prodID %s to %s' % (str(prodID),
",".join(targetSE))
trans.setDescription(description)
trans.setLongDescription(description)
trans.setType('Replication')
trans.setGroup('Replication')
trans.setPlugin('Broadcast')
res = trans.setSourceSE(sourceSE)
if not res['OK']:
exit(1)
res = trans.setTargetSE(targetSE)
if not res['OK']:
exit(1)
res = trans.addTransformation()
if not res['OK']:
gLogger.error(res['Message'])
exit(1)
gLogger.verbose(res)
trans.setStatus('Active')
trans.setAgentType('Automatic')
currtrans = trans.getTransformationID()['Value']
client = TransformationClient()
res = client.createTransformationInputDataQuery(currtrans, metadata)
if res['OK']:
gLogger.always("Successfully created replication transformation")
return S_OK()
else:
gLogger.error("Failure during replication creation", res['Message'])
return S_ERROR("Failed to create transformation")
def DataReplicationTSExample(args=None):
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
if (len(args) != 1):
Script.gLogger.notice('Wrong number of arguments')
Script.showHelp()
infile = args[0]
f = open(infile, 'r')
infileList = []
for line in f:
infile = line.strip()
if line != "\n":
infileList.append(infile)
t = Transformation()
tc = TransformationClient()
t.setTransformationName("DM_Replication") # This must be unique
#t.setTransformationGroup("Group1")
t.setType("Replication")
t.setSourceSE(['CYF-STORM-Disk', 'DESY-ZN-Disk'
]) # A list of SE where at least 1 SE is the valid one
t.setTargetSE(['CEA-Disk'])
t.setDescription("corsika Replication")
t.setLongDescription("corsika Replication") # Mandatory
t.setGroupSize(
1
) # Here you specify how many files should be grouped within the same request, e.g. 100
t.setPlugin("Broadcast") # Mandatory for replication
t.addTransformation() # Transformation is created here
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
tc.addFilesToTransformation(transID['Value'],
infileList) # Files are added here
