def do_getall( self, args ):
"""Get transformation details
usage: getall [Status] [Status]
"""
oTrans = Transformation()
oTrans.getTransformations( transStatus = args.split(), printOutput = True )
def do_getall(self, args):
"""Get transformation details
usage: getall [Status] [Status]
"""
oTrans = Transformation()
oTrans.getTransformations(transStatus=args.split(), printOutput=True)
def do_getall( self, args ):
'''Get transformation details
usage: getall [Status] [Status]
'''
oTrans = Transformation()
oTrans.setServer( self.serverURL )
oTrans.getTransformations( transStatus = args.split(), printOutput = True )
def do_getall( self, args ):
"""Get transformation details
usage: getall [Status] [Status]
"""
oTrans = Transformation()
oTrans.setServer( self.serverURL )
oTrans.getTransformations( transStatus = string.split( args ), printOutput = True )
def test_AddFilesGetFilesSetFileStatus(self):
""" Testing adding, getting and setting file status.
addFilesToTransformation()
getTransformationFiles()
getTransformationStats()
setFileStatusForTransformation()
addTaskForTransformation()
Test adding and files to transformation.
Test selecting the files for the transformation.
Test getting the status count of the transformation files.
Test setting the file status for transformation.
Test creating a task for the added files and ensure the status is updated correctly.
"""
oTrans = Transformation(self.transID)
lfns = ['/test/lfn/file1', '/test/lfn/file2']
res = oTrans.addFilesToTransformation(lfns)
self.assert_(res['OK'])
res = oTrans.getTransformationFiles()
self.assert_(res['OK'])
self.assertEqual(sortList(lfns), res['LFNs'])
self.assertEqual(len(lfns), len(res['Records']))
self.assertEqual(len(lfns), len(res['Value']))
fileParams = sortList([
'LFN', 'TransformationID', 'FileID', 'Status', 'TaskID',
'TargetSE', 'UsedSE', 'ErrorCount', 'LastUpdate', 'InsertedTime'
])
self.assertEqual(fileParams, sortList(res['ParameterNames']))
self.assertEqual(res['Records'][0][0], lfns[0])
self.assertEqual(res['Value'][0]['LFN'], lfns[0])
self.assertEqual(res['Records'][0][1], self.transID)
self.assertEqual(res['Value'][0]['TransformationID'], self.transID)
self.assertEqual(res['Records'][0][3], 'Unused')
self.assertEqual(res['Value'][0]['Status'], 'Unused')
res = oTrans.getTransformationStats()
self.assert_(res['OK'])
self.assertEqual(res['Value']['Total'], 2)
self.assertEqual(res['Value']['Unused'], 2)
res = oTrans.setFileStatusForTransformation('Processed', [lfns[0]])
self.assert_(res['OK'])
res = oTrans.getTransformationStats()
self.assert_(res['OK'])
self.assertEqual(res['Value']['Total'], 2)
self.assertEqual(res['Value']['Unused'], 1)
self.assertEqual(res['Value']['Processed'], 1)
res = oTrans.setFileStatusForTransformation('Unused', [lfns[0]])
self.assert_(res['OK'])
self.assert_(res['Value']['Failed'].has_key(lfns[0]))
self.assertEqual(res['Value']['Failed'][lfns[0]],
'Can not change Processed status')
res = oTrans.addTaskForTransformation(lfns=[lfns[1]], se='Test')
self.assert_(res['OK'])
res = oTrans.getTransformationStats()
self.assert_(res['OK'])
self.assertEqual(res['Value']['Total'], 2)
self.assertEqual(res['Value']['Assigned'], 1)
self.assertEqual(res['Value']['Processed'], 1)
def do_getAllByUser(self, args):
"""Get all transformations created by a given user
The first argument is the authorDN or username. The authorDN
is preferred: it need to be inside quotes because contains
white spaces. Only authorDN should be quoted.
When the username is provided instead,
the authorDN is retrieved from the uploaded proxy,
so that the retrieved transformations are those created by
the user who uploaded that proxy: that user could be different
that the username provided to the function.
usage: getAllByUser authorDN or username [Status] [Status]
"""
oTrans = Transformation()
argss = args.split()
username = ""
author = ""
status = []
if not len(argss) > 0:
print self.do_getAllByUser.__doc__
return
# if the user didnt quoted the authorDN ends
if '=' in argss[0] and argss[0][0] not in ["'", '"']:
print "AuthorDN need to be quoted (just quote that argument)"
return
if argss[0][0] in ["'", '"']: # authorDN given
author = argss[0]
status_idx = 1
for arg in argss[1:]:
author += ' ' + arg
status_idx += 1
if arg[-1] in ["'", '"']:
break
# At this point we should have something like 'author'
if not author[0] in ["'", '"'] or not author[-1] in ["'", '"']:
print "AuthorDN need to be quoted (just quote that argument)"
return
else:
author = author[1:-1] # throw away the quotes
# the rest are the requested status
status = argss[status_idx:]
else: # username given
username = argss[0]
status = argss[1:]
oTrans.getTransformationsByUser(authorDN=author,
userName=username,
transStatus=status,
printOutput=True)
def test_DeleteTransformation(self):
""" Testing of the deletion of a transformation.
addTransformation()
deleteTransformation()
Tests that a transformation can be removed.
Tests that retrieving a non existant transformation raises an AttributeError.
"""
oTrans = Transformation(self.transID)
res = oTrans.deleteTransformation()
self.assert_(res['OK'])
self.assertRaises(AttributeError, Transformation, self.transID)
self.transID = 0
def do_getAllByUser( self, args ):
"""Get all transformations created by a given user
The first argument is the authorDN or username. The authorDN
is preferred: it need to be inside quotes because contains
white spaces. Only authorDN should be quoted.
When the username is provided instead,
the authorDN is retrieved from the uploaded proxy,
so that the retrieved transformations are those created by
the user who uploaded that proxy: that user could be different
that the username provided to the function.
usage: getAllByUser authorDN or username [Status] [Status]
"""
oTrans = Transformation()
argss = args.split()
username = ""
author = ""
status = []
if not len( argss ) > 0:
print self.do_getAllByUser.__doc__
return
# if the user didnt quoted the authorDN ends
if '=' in argss[0] and argss[0][0] not in ["'", '"']:
print "AuthorDN need to be quoted (just quote that argument)"
return
if argss[0][0] in ["'", '"']: # authorDN given
author = argss[0]
status_idx = 1
for arg in argss[1:]:
author += ' ' + arg
status_idx +=1
if arg[-1] in ["'", '"']:
break
# At this point we should have something like 'author'
if not author[0] in ["'", '"'] or not author[-1] in ["'", '"']:
print "AuthorDN need to be quoted (just quote that argument)"
return
else:
author = author[1:-1] # throw away the quotes
# the rest are the requested status
status = argss[ status_idx: ]
else: # username given
username = argss[0]
status = argss[ 1: ]
oTrans.getTransformationsByUser( authorDN = author, userName = username, transStatus = status, printOutput = True )
def test_DeleteTransformation(self):
""" Testing of the deletion of a transformation.
addTransformation()
deleteTransformation()
Tests that a transformation can be removed.
Tests that retrieving a non existant transformation raises an AttributeError.
"""
oTrans = Transformation(self.transID)
res = oTrans.deleteTransformation()
self.assert_(res['OK'])
self.assertRaises(AttributeError,Transformation,self.transID)
self.transID = 0
def test_AddFilesGetFilesSetFileStatus(self):
""" Testing adding, getting and setting file status.
addFilesToTransformation()
getTransformationFiles()
getTransformationStats()
setFileStatusForTransformation()
addTaskForTransformation()
Test adding and files to transformation.
Test selecting the files for the transformation.
Test getting the status count of the transformation files.
Test setting the file status for transformation.
Test creating a task for the added files and ensure the status is updated correctly.
"""
oTrans = Transformation(self.transID)
lfns = ['/test/lfn/file1','/test/lfn/file2']
res = oTrans.addFilesToTransformation(lfns)
self.assert_(res['OK'])
res = oTrans.getTransformationFiles()
self.assert_(res['OK'])
self.assertEqual(sortList(lfns),res['LFNs'])
self.assertEqual(len(lfns),len(res['Records']))
self.assertEqual(len(lfns),len(res['Value']))
fileParams = sortList(['LFN', 'TransformationID', 'FileID', 'Status', 'TaskID', 'TargetSE', 'UsedSE', 'ErrorCount', 'LastUpdate', 'InsertedTime'])
self.assertEqual(fileParams,sortList(res['ParameterNames']))
self.assertEqual(res['Records'][0][0], lfns[0])
self.assertEqual(res['Value'][0]['LFN'],lfns[0])
self.assertEqual(res['Records'][0][1], self.transID)
self.assertEqual(res['Value'][0]['TransformationID'], self.transID)
self.assertEqual(res['Records'][0][3],'Unused')
self.assertEqual(res['Value'][0]['Status'],'Unused')
res = oTrans.getTransformationStats()
self.assert_(res['OK'])
self.assertEqual(res['Value']['Total'],2)
self.assertEqual(res['Value']['Unused'],2)
res = oTrans.setFileStatusForTransformation('Processed',[lfns[0]])
self.assert_(res['OK'])
res = oTrans.getTransformationStats()
self.assert_(res['OK'])
self.assertEqual(res['Value']['Total'],2)
self.assertEqual(res['Value']['Unused'],1)
self.assertEqual(res['Value']['Processed'],1)
res = oTrans.setFileStatusForTransformation('Unused',[lfns[0]])
self.assert_(res['OK'])
self.assert_(res['Value']['Failed'].has_key(lfns[0]))
self.assertEqual(res['Value']['Failed'][lfns[0]],'Can not change Processed status')
res = oTrans.addTaskForTransformation(lfns=[lfns[1]],se='Test')
self.assert_(res['OK'])
res = oTrans.getTransformationStats()
self.assert_(res['OK'])
self.assertEqual(res['Value']['Total'],2)
self.assertEqual(res['Value']['Assigned'],1)
self.assertEqual(res['Value']['Processed'],1)
def do_summaryTransformations( self, args ):
"""Show the summary for a list of Transformations
Fields starting with 'F' ('J') refers to files (jobs).
Proc. stand for processed.
Usage: summaryTransformations <ProdID> [<ProdID> ...]
"""
argss = args.split()
if not len( argss ) > 0:
print self.do_summaryTransformations.__doc__
return
transid = argss
oTrans = Transformation()
oTrans.getSummaryTransformations( transID = transid )
def setUp( self ):
self.mockTransClient = Mock()
self.mockTransClient.setTaskStatusAndWmsID.return_value = {'OK':True}
self.WMSClientMock = Mock()
self.jobMonitoringClient = Mock()
self.mockReqClient = Mock()
self.jobMock = Mock()
self.jobMock2 = Mock()
mockWF = Mock()
mockPar = Mock()
mockWF.findParameter.return_value = mockPar
mockPar.getValue.return_value = 'MySite'
self.jobMock2.workflow = mockWF
self.jobMock2.setDestination.return_value = {'OK':True}
self.jobMock.workflow.return_value = ''
self.jobMock.return_value = self.jobMock2
self.taskBase = TaskBase( transClient = self.mockTransClient )
self.wfTasks = WorkflowTasks( transClient = self.mockTransClient,
submissionClient = self.WMSClientMock,
jobMonitoringClient = self.jobMonitoringClient,
outputDataModule = "mock",
jobClass = self.jobMock )
self.requestTasks = RequestTasks( transClient = self.mockTransClient,
requestClient = self.mockReqClient
)
self.tc = TransformationClient()
self.transformation = Transformation()
self.maxDiff = None
def __init__(self):
"""Create a string containing restructured text documentation for all the special tranformation parameters
:param str module: full path to the module
:param bool replace: wether to replace the full docstring or not
if you replace it completely add the automodule commands etc.!
"""
self.module = MODULE
self.replace = False
self.doc_string = ''
# Fill the docstring addition with what we want to add
self.doc_string += textwrap.dedent("""
Transformation Parameters
-------------------------
Any parameter with ``ParameterName`` can be set for a transformation with a call
to ``setParameterName(parameterValue)``.
The following parameters have a special meaning
""")
from DIRAC.TransformationSystem.Client.Transformation import Transformation
trans = Transformation()
for paramName in sorted(trans.paramTypes):
self.doc_string += '\n``%s``:\n Default value: %r' % (
paramName, trans.paramValues[paramName])
def do_summaryTransformations(self, args):
"""Show the summary for a list of Transformations
Fields starting with 'F' ('J') refers to files (jobs).
Proc. stand for processed.
Usage: summaryTransformations <ProdID> [<ProdID> ...]
"""
argss = args.split()
if not len(argss) > 0:
print self.do_summaryTransformations.__doc__
return
transid = argss
oTrans = Transformation()
oTrans.getSummaryTransformations(transID=transid)
def test_SetGetReset(self):
""" Testing of the set, get and reset methods.
set*()
get*()
setTargetSE()
setSourceSE()
getTargetSE()
getSourceSE()
reset()
Ensures that after a reset all parameters are returned to their defaults
"""
oTrans = Transformation()
res = oTrans.getParameters()
self.assert_(res['OK'])
defaultParams = res['Value'].copy()
for parameterName, defaultValue in res['Value'].items():
if type(defaultValue) in types.StringTypes:
testValue = "'TestValue'"
else:
testValue = '99999'
execString = "res = oTrans.set%s(%s)" % (parameterName, testValue)
exec(execString)
self.assert_(res['OK'])
execString = "res = oTrans.get%s()" % (parameterName)
exec(execString)
self.assert_(res['OK'])
self.assertEqual(res['Value'], eval(testValue))
# Test that SEs delivered as a space or comma seperated string are resolved...
stringSEs = 'CERN-USER, CNAF-USER GRIDKA-USER,IN2P3-USER'
listSEs = stringSEs.replace(',', ' ').split()
res = oTrans.setTargetSE(stringSEs)
self.assert_(res['OK'])
res = oTrans.getTargetSE()
self.assert_(res['OK'])
self.assertEqual(res['Value'], listSEs)
# ...but that lists are correctly handled also
res = oTrans.setSourceSE(listSEs)
self.assert_(res['OK'])
res = oTrans.getSourceSE()
self.assert_(res['OK'])
self.assertEqual(res['Value'], listSEs)
res = oTrans.reset()
self.assert_(res['OK'])
res = oTrans.getParameters()
self.assert_(res['OK'])
for parameterName, resetValue in res['Value'].items():
self.assertEqual(resetValue, defaultParams[parameterName])
self.assertRaises(AttributeError, oTrans.getTargetSE)
self.assertRaises(AttributeError, oTrans.getSourceSE)
def test_SetGetReset(self):
""" Testing of the set, get and reset methods.
set*()
get*()
setTargetSE()
setSourceSE()
getTargetSE()
getSourceSE()
reset()
Ensures that after a reset all parameters are returned to their defaults
"""
oTrans = Transformation()
res = oTrans.getParameters()
self.assert_(res['OK'])
defaultParams = res['Value'].copy()
for parameterName, defaultValue in res['Value'].items():
if type(defaultValue) in types.StringTypes:
testValue = "'TestValue'"
else:
testValue = '99999'
execString = "res = oTrans.set%s(%s)" % (parameterName,testValue)
exec(execString)
self.assert_(res['OK'])
execString = "res = oTrans.get%s()" % (parameterName)
exec(execString)
self.assert_(res['OK'])
self.assertEqual(res['Value'],eval(testValue))
# Test that SEs delivered as a space or comma seperated string are resolved...
stringSEs = 'CERN-USER, CNAF-USER GRIDKA-USER,IN2P3-USER'
listSEs = stringSEs.replace(',',' ').split()
res = oTrans.setTargetSE(stringSEs)
self.assert_(res['OK'])
res = oTrans.getTargetSE()
self.assert_(res['OK'])
self.assertEqual(res['Value'],listSEs)
# ...but that lists are correctly handled also
res = oTrans.setSourceSE(listSEs)
self.assert_(res['OK'])
res = oTrans.getSourceSE()
self.assert_(res['OK'])
self.assertEqual(res['Value'],listSEs)
res = oTrans.reset()
self.assert_(res['OK'])
res = oTrans.getParameters()
self.assert_(res['OK'])
for parameterName, resetValue in res['Value'].items():
self.assertEqual(resetValue,defaultParams[parameterName])
self.assertRaises(AttributeError,oTrans.getTargetSE)
self.assertRaises(AttributeError,oTrans.getSourceSE)
def test_getTransformations(self):
""" Testing the selection of transformations from the database
getTransformations
This will select all the transformations associated to this test suite and remove them.
"""
oTrans = Transformation()
res = oTrans.getTransformations()
self.assert_(res['OK'])
parameters = [
'TransformationID', 'TransformationName', 'Description',
'LongDescription', 'CreationDate', 'LastUpdate', 'AuthorDN',
'AuthorGroup', 'Type', 'Plugin', 'AgentType', 'Status', 'FileMask',
'TransformationGroup', 'GroupSize', 'InheritedFrom', 'Body',
'MaxNumberOfTasks', 'EventsPerTask'
]
self.assertEqual(sortList(res['ParameterNames']), sortList(parameters))
self.assertEqual(sortList(res['Value'][0].keys()),
sortList(parameters))
self.assertEqual(len(res['Value']), len(res['Records']))
ignore = self.transID
for transDict in res['Value']:
name = transDict['TransformationName']
if re.search('TestClientTransformation', name):
transID = transDict['TransformationID']
if transID != ignore:
oTrans = Transformation(transID)
res = oTrans.deleteTransformation()
self.assert_(res['OK'])
self.transID = ignore
def do_getall( self, args ):
"""Get transformation details
usage: getall [Status] [Status]
"""
oTrans = Transformation()
oTrans.setServer( self.serverURL )
oTrans.getTransformations( transStatus = string.split( args ), printOutput = True )
def do_getall(self, args):
'''Get transformation details
usage: getall [Status] [Status]
'''
oTrans = Transformation()
oTrans.setServer(self.serverURL)
oTrans.getTransformations(transStatus=args.split(), printOutput=True)
def test_getTransformations(self):
""" Testing the selection of transformations from the database
getTransformations
This will select all the transformations associated to this test suite and remove them.
"""
oTrans = Transformation()
res = oTrans.getTransformations()
self.assert_(res['OK'])
parameters = ['TransformationID', 'TransformationName', 'Description', 'LongDescription', 'CreationDate', 'LastUpdate', 'AuthorDN', 'AuthorGroup', 'Type', 'Plugin', 'AgentType', 'Status', 'FileMask', 'TransformationGroup', 'GroupSize', 'InheritedFrom', 'Body', 'MaxNumberOfTasks', 'EventsPerTask']
self.assertEqual(sortList(res['ParameterNames']),sortList(parameters))
self.assertEqual(sortList(res['Value'][0].keys()),sortList(parameters))
self.assertEqual(len(res['Value']),len(res['Records']))
ignore = self.transID
for transDict in res['Value']:
name = transDict['TransformationName']
if re.search('TestClientTransformation',name):
transID = transDict['TransformationID']
if transID != ignore:
oTrans = Transformation(transID)
res = oTrans.deleteTransformation()
self.assert_(res['OK'])
self.transID = ignore
def _createTransformation(self, transName, plugin='Standard'):
oTrans = Transformation()
res = oTrans.setTransformationName(transName)
self.assert_(res['OK'])
description = 'Test transforamtion description'
res = oTrans.setDescription(description)
longDescription = 'Test transformation long description'
res = oTrans.setLongDescription(longDescription)
self.assert_(res['OK'])
res = oTrans.setType('MCSimulation')
self.assert_(res['OK'])
res = oTrans.setPlugin(plugin)
self.assert_(res['OK'])
res = oTrans.addTransformation()
self.assert_(res['OK'])
self.transID = res['Value']
return res
def setUp(self):
self.mockTransClient = mock.MagicMock()
self.mockTransClient.setTaskStatusAndWmsID.return_value = {'OK': True}
self.mockReqClient = mock.MagicMock()
self.taskBase = TaskBase(transClient=self.mockTransClient)
self.pu = PluginUtilities(transClient=self.mockTransClient)
self.requestTasks = RequestTasks(transClient=self.mockTransClient,
requestClient=self.mockReqClient,
requestValidator=reqValFake)
self.transformation = Transformation()
self.maxDiff = None
def test_getTransformationLogging(self):
""" Testing the obtaining of transformation logging information
getTransformationLogging()
"""
oTrans = Transformation(self.transID)
res = oTrans.setStatus('Active')
self.assert_(res['OK'])
res = oTrans.extendTransformation(100)
self.assert_(res['OK'])
res = oTrans.setStatus('Completed')
self.assert_(res['OK'])
res = oTrans.cleanTransformation()
self.assert_(res['OK'])
res = oTrans.getTransformationLogging()
self.assert_(res['OK'])
self.assertEqual(len(res['Value']), 6)
def _createTransformation(self,transName,plugin='Standard'):
oTrans = Transformation()
res = oTrans.setTransformationName(transName)
self.assert_(res['OK'])
description = 'Test transforamtion description'
res = oTrans.setDescription(description)
longDescription = 'Test transformation long description'
res = oTrans.setLongDescription(longDescription)
self.assert_(res['OK'])
res = oTrans.setType('MCSimulation')
self.assert_(res['OK'])
res = oTrans.setPlugin(plugin)
self.assert_(res['OK'])
res = oTrans.addTransformation()
self.assert_(res['OK'])
self.transID = res['Value']
return res
def setUp(self):
self.mockTransClient = MagicMock()
self.mockTransClient.setTaskStatusAndWmsID.return_value = {'OK': True}
self.WMSClientMock = MagicMock()
self.jobMonitoringClient = MagicMock()
self.mockReqClient = MagicMock()
self.jobMock = MagicMock()
self.jobMock2 = MagicMock()
mockWF = MagicMock()
mockPar = MagicMock()
mockWF.findParameter.return_value = mockPar
mockPar.getValue.return_value = 'MySite'
self.jobMock2.workflow = mockWF
self.jobMock2.setDestination.return_value = {'OK': True}
self.jobMock.workflow.return_value = ''
self.jobMock.return_value = self.jobMock2
self.reqValidatorMock = MagicMock()
self.reqValidatorMock.validate.return_value = {'OK': True}
self.taskBase = TaskBase(transClient=self.mockTransClient)
self.pu = PluginUtilities(transClient=self.mockTransClient)
self.wfTasks = WorkflowTasks(
transClient=self.mockTransClient,
submissionClient=self.WMSClientMock,
jobMonitoringClient=self.jobMonitoringClient,
outputDataModule="mock")
self.requestTasks = RequestTasks(transClient=self.mockTransClient,
requestClient=self.mockReqClient,
requestValidator=reqValFake)
self.tc = TransformationClient()
self.transformation = Transformation()
self.maxDiff = None
gLogger.setLevel('DEBUG')
def test_getTransformationLogging(self):
""" Testing the obtaining of transformation logging information
getTransformationLogging()
"""
oTrans = Transformation(self.transID)
res = oTrans.setStatus('Active')
self.assert_(res['OK'])
res = oTrans.extendTransformation(100)
self.assert_(res['OK'])
res = oTrans.setStatus('Completed')
self.assert_(res['OK'])
res = oTrans.cleanTransformation()
self.assert_(res['OK'])
res = oTrans.getTransformationLogging()
self.assert_(res['OK'])
self.assertEqual(len(res['Value']),6)
def OldSubmitTS(job, infileList):
""" Create a transformation executing the job workflow """
t = Transformation()
tc = TransformationClient()
t.setType("DataReprocessing")
t.setDescription("Runs EvnDisp analysis for array HB9 SCT")
t.setLongDescription(
"merge_simtel, evndisp converter and evndisp analysis for HB9 SCT"
) # mandatory
t.setGroupSize(5)
t.setBody(job.workflow.toXML())
res = t.addTransformation() # Transformation is created here
if not res['OK']:
print res['Message']
DIRAC.exit(-1)
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
print('Adding %s files to transformation' % len(infileList))
tc.addFilesToTransformation(transID['Value'],
infileList) # Files added here
return res
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
from DIRAC.Interfaces.API.Job import Job
j = Job()
tc = TransformationClient()
t = Transformation()
t.setTransformationName("Un exemple") #<- unique
t.setTransformationGroup("Un groupe") #<- for monitoring
t.setType("MCSimulation")#type must be among known types
t.setDescription("Ceci est un exemple")
t.setLongDescription("C'est un bel exemple")
t.setBody(j.workflow._toXML())
t.setGroupSize(1)
t.setPlugin("Standard")
t.addTransformation() #<-- transformation is created here
t.setStatus("Active") #<-- make it start
t.setAgentType("Automatic") #<-- should be by default
transfid = t.getTransformationID()['Value'] #<- unique
tc.createTransformationInputDataQuery(transfid,
{'meta1':val1,"meta2":{">":34}})
if not res['OK']:
print res['Message']
res = psl.finalizeProd()
if not res['OK']:
print res['Message']
exit(1)
# get the metadata for this production to define input for the next step
meta = psl.getMetadata()
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
# Define transformation steps for the replication of the output data
if replicateFiles and meta:
Trans = Transformation()
Trans.setTransformationName( 'replicate_%s_%s_%s_%s' % ( process, energy, polarisation, meta['Datatype'] ) )
description = 'Replicate %s %s %s %s to' % ( process, energy, polarisation, meta['Datatype'] )
for replicaSRM in replicaSRMs:
description += ' %s,' % ( replicaSRM )
description.rstrip( ',' )
Trans.setDescription( description )
Trans.setLongDescription( description )
Trans.setType( 'Replication' )
Trans.setPlugin( 'Broadcast' )
Trans.setSourceSE( outputSRM )
Trans.setTargetSE( replicaSRMs )
res = Trans.addTransformation()
if not res['OK']:
print res
name = prodJob.workflow.getName()
name = name.replace('/', '').replace('\\', '')
prodJob.workflow.toXMLFile(name)
print 'Workflow XML file name is: %s' % name
workflowBody = ''
if os.path.exists(name):
with open(name, 'r') as fopen:
workflowBody = fopen.read()
else:
print 'Could not get workflow body'
# Standard parameters
transformation = Transformation()
transformation.setTransformationName(name)
transformation.setTransformationGroup('Test')
transformation.setDescription(desc)
transformation.setLongDescription(desc)
transformation.setType('Merge')
transformation.setBody(workflowBody)
transformation.setPlugin('Standard')
transformation.setTransformationFamily('Test')
transformation.setGroupSize(2)
transformation.setOutputDirectories(['/dirac/outConfigName/configVersion/LOG/00000000',
'/dirac/outConfigName/configVersion/RAW/00000000',
'/dirac/outConfigName/configVersion/CORE/00000000'])
# Set directory meta data and create a transformation with a meta-data filter
def submitTS(job):
""" Create a transformation executing the job workflow """
### Temporary fix to initialize JOB_ID #######
job.workflow.addParameter(
Parameter("JOB_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
job.workflow.addParameter(
Parameter("PRODUCTION_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
job.setType('MCSimulation') ## Used for the JobType plugin
t = Transformation()
# t.setTransformationName("Prod3Exemple") # This must be unique. If not set it's asked in the prompt
t.setType("MCSimulation")
t.setDescription("MC Prod3 BaseLine HE test")
t.setLongDescription("corsika-simtel production") # mandatory
t.setBody(job.workflow.toXML())
res = t.addTransformation() # Transformation is created here
if not res['OK']:
print(res['Message'])
DIRAC.exit(-1)
t.setStatus("Active")
t.setAgentType("Automatic")
return res
def createDataTransformation(
flavour,
targetSE,
sourceSE,
metaKey,
metaValue,
extraData=None,
extraname="",
groupSize=1,
plugin="Broadcast",
tGroup=None,
tBody=None,
enable=False,
):
"""Creates the replication transformation based on the given parameters.
:param str flavour: Flavour of replication to create: Replication or Moving
:param targetSE: Destination for files
:type targetSE: python:list or str
:param sourceSE: Origin of files
:type sourceSE: python:list or str
:param int metaKey: Meta key to identify input files
:param int metaValue: Meta value to identify input files
:param dict metaData: Additional meta data to use to identify input files
:param str extraname: addition to the transformation name, only needed if the same transformation was already created
:param int groupSize: number of files per transformation taks
:param str plugin: plugin to use
:param str tGroup: transformation group to set
:param tBody: transformation body to set
:param bool enable: if true submit the transformation, otherwise dry run
:returns: S_OK (with the transformation object, if successfully added), S_ERROR
"""
metadata = {metaKey: metaValue}
if isinstance(extraData, dict):
metadata.update(extraData)
gLogger.debug("Using %r for metadata search" % metadata)
if isinstance(targetSE, six.string_types):
targetSE = [targetSE]
if isinstance(sourceSE, (list, tuple)):
sourceSE = "%s" % (",".join(sourceSE))
gLogger.debug("Using plugin: %r" % plugin)
if flavour not in ("Replication", "Moving"):
return S_ERROR("Unsupported flavour %s" % flavour)
transVerb = {"Replication": "Replicate", "Moving": "Move"}[flavour]
transGroup = {
"Replication": "Replication",
"Moving": "Moving"
}[flavour] if not tGroup else tGroup
trans = Transformation()
transName = "%s_%s_%s" % (transVerb, str(metaValue), ",".join(targetSE))
if extraname:
transName += "_%s" % extraname
trans.setTransformationName(transName)
description = "%s files for %s %s to %s" % (
transVerb, metaKey, str(metaValue), ",".join(targetSE))
trans.setDescription(description[:255])
trans.setLongDescription(description)
trans.setType("Replication")
trans.setTransformationGroup(transGroup)
trans.setGroupSize(groupSize)
trans.setPlugin(plugin)
transBody = ({
"Moving": [
("ReplicateAndRegister", {
"SourceSE": sourceSE,
"TargetSE": targetSE
}),
("RemoveReplica", {
"TargetSE": sourceSE
}),
],
"Replication":
"", # empty body
}[flavour] if tBody is None else tBody)
trans.setBody(transBody)
trans.setInputMetaQuery(metadata)
if sourceSE:
res = trans.setSourceSE(sourceSE)
if not res["OK"]:
return S_ERROR("SourceSE not valid: %s" % res["Message"])
res = trans.setTargetSE(targetSE)
if not res["OK"]:
return S_ERROR("TargetSE not valid: %s" % res["Message"])
if not enable:
gLogger.always("Dry run, not creating transformation")
return S_OK()
res = trans.addTransformation()
if not res["OK"]:
return res
gLogger.verbose(res)
trans.setStatus("Active")
trans.setAgentType("Automatic")
gLogger.always("Successfully created replication transformation")
return S_OK(trans)
def submit_trans(job, trans_name, mqJson, group_size, with_file_mask=True):
""" Create a transformation executing the job workflow
"""
DIRAC.gLogger.notice('submit_trans : %s' % trans_name)
# Initialize JOB_ID
job.workflow.addParameter(
Parameter("JOB_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
trans = Transformation()
trans.setTransformationName(trans_name) # this must be unique
trans.setType("DataReprocessing")
trans.setDescription("Prod3 DL1 Data Handler TS")
trans.setLongDescription("Prod3 DL1 Data Handler conversion") # mandatory
trans.setBody(job.workflow.toXML())
trans.setGroupSize(group_size)
if with_file_mask:
trans.setFileMask(mqJson) # catalog query is defined here
result = trans.addTransformation() # transformation is created here
if not result['OK']:
return result
trans.setStatus("Active")
trans.setAgentType("Automatic")
trans_id = trans.getTransformationID()
return trans_id
if not result['OK']:
gLogger.error("Error: %s" % result['Message'])
exit(0)
if not result['Value']:
gLogger.error("Error: no metadata fields defined")
exit(0)
typeDict = result['Value']['FileMetaFields']
typeDict.update(result['Value']['DirectoryMetaFields'])
# Special meta tags
typeDict.update(FILE_STANDARD_METAKEYS)
mq = MetaQuery(typeDict=typeDict)
mq.setMetaQuery([metaTransfer])
query = mq.getMetaQuery()
gLogger.notice('Query: {0}'.format(query))
t = Transformation()
tc = TransformationClient()
t.setTransformationName(transformationName) # Must be unique
t.setTransformationGroup("Transfer")
t.setType(transformationType)
t.setPlugin(plugin)
t.setDescription("Data Transfer")
t.setLongDescription("Data Transfer") # Mandatory
t.setGroupSize(
groupSize
) # Here you specify how many files should be grouped within he same request, e.g. 100
transBody = ''
t.setBody(transBody)
if sourceSE:
def test_ExtendCleanTransformation(self):
""" Tests the extension of transformations and the removal of tasks. Also obtain tasks, their status and update their status.
extendTransformation()
getTransformationTasks()
getTransformationTaskStats()
deleteTasks()
setTaskStatus()
cleanTransformation()
Tests a transformation can be extended.
Tests can obtain the the transformation tasks and their statistics.
Tests the removal of already created tasks.
Tests can change the status of a task.
Tests that Cleaning a transformation removes tasks defined for the transformation.
"""
oTrans = Transformation(self.transID)
nTasks = 100
res = oTrans.extendTransformation(nTasks)
self.assert_(res['OK'])
taskIDs = res['Value']
self.assertEqual(len(taskIDs),nTasks)
res = oTrans.getTransformationTasks()
self.assert_(res['OK'])
parameters = ['TargetSE', 'TransformationID', 'LastUpdateTime', 'ExternalID', 'CreationTime', 'TaskID', 'ExternalStatus']
self.assertEqual(sortList(res['ParameterNames']),sortList(parameters))
self.assertEqual(sortList(res['Value'][0].keys()),sortList(parameters))
self.assertEqual(res['Value'][0]['TargetSE'],'Unknown')
self.assertEqual(res['Value'][0]['TransformationID'],self.transID)
self.assertEqual(res['Value'][0]['ExternalID'],'0')
self.assertEqual(res['Value'][0]['TaskID'],1)
self.assertEqual(res['Value'][0]['ExternalStatus'],'Created')
self.assertEqual(res['Records'][0][0],1)
self.assertEqual(res['Records'][0][1],self.transID)
self.assertEqual(res['Records'][0][2],'Created')
self.assertEqual(res['Records'][0][3],'0')
self.assertEqual(res['Records'][0][4],'Unknown')
res = oTrans.getTransformationTaskStats()
self.assert_(res['OK'])
self.assertEqual(res['Value']['Created'],100)
res = oTrans.deleteTasks(11,100)
self.assert_(res['OK'])
res = oTrans.getTransformationTaskStats()
self.assert_(res['OK'])
self.assertEqual(res['Value']['Created'],10)
res = oTrans.setTaskStatus(1, 'Done')
self.assert_(res['OK'])
res = oTrans.getTransformationTaskStats()
self.assert_(res['OK'])
self.assertEqual(res['Value']['Created'],10)
self.assertEqual(res['Value']['Done'],1)
res = oTrans.cleanTransformation()
self.assert_(res['OK'])
res = oTrans.getStatus()
self.assert_(res['OK'])
self.assertEqual(res['Value'],'Cleaned')
res = oTrans.getTransformationTasks()
self.assert_(res['OK'])
self.assertFalse(res['Value'])
self.assertFalse(res['Records'])
def createMovingTransformation(targetSE,
sourceSE,
prodID,
datatype,
extraname='',
forceMoving=False):
"""Creates the replication transformation based on the given parameters
:param targetSE: Destination for files
:type targetSE: python:list or str
:param str sourceSE: Origin of files. Files will be removed from this SE
:param int prodID: Production ID of files to be moved
:param str datatype: DataType of files to be moved
:param str extraname: addition to the transformation name, only needed if the same transformation was already created
:param bool forceMoving: Move always, even if GEN/SIM files don't have descendents
:returns: S_OK, S_ERROR
"""
metadata = {"Datatype": datatype, "ProdID": prodID}
if isinstance(targetSE, basestring):
targetSE = [targetSE]
trans = Transformation()
transName = 'Move_%s_%s_%s' % (datatype, str(prodID), ",".join(targetSE))
if extraname:
transName += "_%s" % extraname
trans.setTransformationName(transName)
description = 'Move files for prodID %s to %s' % (str(prodID),
",".join(targetSE))
trans.setDescription(description)
trans.setLongDescription(description)
trans.setType('Replication')
trans.setGroup('Moving')
if datatype in ('GEN', 'SIM') and not forceMoving:
trans.setPlugin('BroadcastProcessed')
else:
trans.setPlugin('Broadcast')
transBody = [
("ReplicateAndRegister", {
"SourceSE": sourceSE,
"TargetSE": targetSE
}),
("RemoveReplica", {
"TargetSE": sourceSE
}),
]
trans.setBody(transBody)
res = trans.setSourceSE(sourceSE)
if not res['OK']:
return S_ERROR("SourceSE not valid: %s" % res['Message'])
res = trans.setTargetSE(targetSE)
if not res['OK']:
return S_ERROR("TargetSE not valid: %s" % res['Message'])
res = trans.addTransformation()
if not res['OK']:
gLogger.error("Failed to create Transformation", res['Message'])
return res
gLogger.verbose(res)
trans.setStatus('Active')
trans.setAgentType('Automatic')
currtrans = trans.getTransformationID()['Value']
client = TransformationClient()
res = client.createTransformationInputDataQuery(currtrans, metadata)
if res['OK']:
gLogger.always("Successfully created replication transformation")
return S_OK()
else:
gLogger.error("Failure during replication creation", res['Message'])
return S_ERROR("Failed to create transformation:%s " % res['Message'])
exit(1)
print("Example file: %s" % lfns[0])
answer = raw_input('Proceed and submit replication? (Y/N): ')
if not answer.lower() in ('y', 'yes'):
print("Canceled")
exit(1)
trc = TransformationClient()
res = trc.getTransformationStats(name_of_replication)
if res['OK']:
print("Replication with name %s already exists! Cannot proceed." % name_of_replication)
exit(1)
Trans = Transformation()
Trans.setTransformationName(name_of_replication)
Trans.setDescription(description)
Trans.setLongDescription(description)
Trans.setType('Replication')
Trans.setPlugin('Broadcast')
#Trans.setFileMask(fmask)
Trans.setSourceSE(source)
Trans.setTargetSE(destination)
res = Trans.addTransformation()
if not res['OK']:
print("Failed to add Replication: %s" % res['Message'])
exit(1)
Trans.setStatus("Active")
Trans.setAgentType("Automatic")
currtrans = Trans.getTransformationID()['Value']
def createDataTransformation(flavour, targetSE, sourceSE,
metaKey, metaValue,
extraData=None, extraname='',
groupSize=1,
plugin='Broadcast',
tGroup=None,
tBody=None,
enable=False,
):
"""Creates the replication transformation based on the given parameters.
:param str flavour: Flavour of replication to create: Replication or Moving
:param targetSE: Destination for files
:type targetSE: python:list or str
:param str sourceSE: Origin of files.
:param int metaKey: Meta key to identify input files
:param int metaValue: Meta value to identify input files
:param dict metaData: Additional meta data to use to identify input files
:param str extraname: addition to the transformation name, only needed if the same transformation was already created
:param int groupSize: number of files per transformation taks
:param str plugin: plugin to use
:param str tGroup: transformation group to set
:param tBody: transformation body to set
:param bool enable: if true submit the transformation, otherwise dry run
:returns: S_OK (with the transformation object, if successfully added), S_ERROR
"""
metadata = {metaKey: metaValue}
if isinstance(extraData, dict):
metadata.update(extraData)
gLogger.debug("Using %r for metadata search" % metadata)
if isinstance(targetSE, basestring):
targetSE = [targetSE]
if flavour not in ('Replication', 'Moving'):
return S_ERROR('Unsupported flavour %s' % flavour)
transVerb = {'Replication': 'Replicate', 'Moving': 'Move'}[flavour]
transGroup = {'Replication': 'Replication', 'Moving': 'Moving'}[flavour] if not tGroup else tGroup
trans = Transformation()
transName = '%s_%s_%s' % (transVerb, str(metaValue), ",".join(targetSE))
if extraname:
transName += "_%s" % extraname
trans.setTransformationName(transName)
description = '%s files for %s %s to %s' % (transVerb, metaKey, str(metaValue), ",".join(targetSE))
trans.setDescription(description)
trans.setLongDescription(description)
trans.setType('Replication')
trans.setTransformationGroup(transGroup)
trans.setGroupSize(groupSize)
trans.setPlugin(plugin)
transBody = {'Moving': [("ReplicateAndRegister", {"SourceSE": sourceSE, "TargetSE": targetSE}),
("RemoveReplica", {"TargetSE": sourceSE})],
'Replication': '', # empty body
}[flavour] if tBody is None else tBody
trans.setBody(transBody)
if sourceSE:
res = trans.setSourceSE(sourceSE)
if not res['OK']:
return S_ERROR("SourceSE not valid: %s" % res['Message'])
res = trans.setTargetSE(targetSE)
if not res['OK']:
return S_ERROR("TargetSE not valid: %s" % res['Message'])
if not enable:
gLogger.always("Dry run, not creating transformation")
return S_OK()
res = trans.addTransformation()
if not res['OK']:
return res
gLogger.verbose(res)
trans.setStatus('Active')
trans.setAgentType('Automatic')
currtrans = trans.getTransformationID()['Value']
client = TransformationClient()
res = client.createTransformationInputDataQuery(currtrans, metadata)
if not res['OK']:
return res
gLogger.always("Successfully created replication transformation")
return S_OK(trans)
def submitTS( job, infileList ):
""" Create a transformation executing the job workflow """
t = Transformation()
tc = TransformationClient()
t.setType( "SimtelMerging" )
t.setDescription( "Runs merge_simtel for array 3HB89" )
t.setLongDescription( "Merging array 3HB89 analysis" ) # mandatory
t.setGroupSize(5)
t.setBody ( job.workflow.toXML() )
res = t.addTransformation() # Transformation is created here
if not res['OK']:
print res['Message']
DIRAC.exit( -1 )
t.setStatus( "Active" )
t.setAgentType( "Automatic" )
transID = t.getTransformationID()
print('Adding %s files to transformation'%len(infileList))
tc.addFilesToTransformation( transID['Value'], infileList ) # Files added here
return res
def submit_trans(job, trans_name, input_meta_query, group_size):
""" Create a transformation executing the job workflow
"""
DIRAC.gLogger.notice('submit_trans : %s' % trans_name)
# Initialize JOB_ID
job.workflow.addParameter(
Parameter("JOB_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
trans = Transformation()
trans.setTransformationName(trans_name) # this must be unique
trans.setType("DataReprocessing")
trans.setDescription("Prod5 EventDisplay TS")
trans.setLongDescription("Prod5 EventDisplay processing") # mandatory
trans.setBody(job.workflow.toXML())
trans.setGroupSize(group_size)
trans.setInputMetaQuery(input_meta_query)
result = trans.addTransformation() # transformation is created here
if not result['OK']:
return result
trans.setStatus("Active")
trans.setAgentType("Automatic")
trans_id = trans.getTransformationID()
return trans_id
def tearDown(self):
if self.transID:
oTrans = Transformation(self.transID)
oTrans.deleteTransformation()
def submit_trans(job, trans_name):
""" Create a transformation executing the job workflow
"""
DIRAC.gLogger.notice('submit_trans : %s' % trans_name)
# Initialize JOB_ID
job.workflow.addParameter(Parameter("JOB_ID", "000000", "string", "", "",
True, False, "Temporary fix"))
trans = Transformation()
trans.setTransformationName(trans_name) # this must be unique
trans.setType("MCSimulation")
trans.setDescription("Prod5 MC Pipe NSB TS")
trans.setLongDescription("Prod5 simulation pipeline") # mandatory
trans.setBody(job.workflow.toXML())
result = trans.addTransformation() # transformation is created here
if not result['OK']:
return result
trans.setStatus("Active")
trans.setAgentType("Automatic")
trans_id = trans.getTransformationID()
return trans_id
def submit_trans(job, infileList, trans_name, group_size):
""" Create a transformation executing the job workflow
"""
DIRAC.gLogger.notice('submit_trans : %s' % trans_name)
# Initialize JOB_ID
job.workflow.addParameter(
Parameter("JOB_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
trans = Transformation()
trans.setTransformationName(trans_name) # this must be unique
trans.setType("DataReprocessing")
trans.setDescription("Simtel TS example")
trans.setLongDescription("Simtel tel_sim") # mandatory
trans.setBody(job.workflow.toXML())
trans.setGroupSize(group_size)
res = trans.addTransformation() # transformation is created here
if not res['OK']:
return res
trans.setStatus("Active")
trans.setAgentType("Automatic")
# add 10*group_size files to transformation (to have the first 10 jobs)
trans_id = trans.getTransformationID()
trans_client = TransformationClient()
res = trans_client.addFilesToTransformation(trans_id['Value'],
infileList[:10 * group_size])
return res
def submit_trans(job, input_meta_query, group_size):
""" Create a transformation executing the job workflow """
#DIRAC.gLogger.notice('submit_trans : %s' % trans_name)
# Initialize JOB_ID
job.workflow.addParameter(
Parameter("JOB_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
t = Transformation()
t.setType("DataReprocessing")
t.setDescription("EvnDisplay MQ example")
t.setLongDescription("EvnDisplay calib_imgreco") # mandatory
t.setBody(job.workflow.toXML())
t.setGroupSize(group_size)
t.setInputMetaQuery(input_meta_query)
res = t.addTransformation() # transformation is created here
if not res['OK']:
return res
#t.setStatus("Active")
t.setAgentType("Automatic")
trans_id = t.getTransformationID()
return trans_id
def submitTS():
########################################
# Modify here with your dirac username
owner = 'user02'
########################################
########################################
# Job description
########################################
job = Job()
job.setName('merge mandelbrot')
job.setOutputSandbox(['*log'])
## define the job workflow in 3 steps
# job step1: setup software
job.setExecutable('git clone https://github.com/bregeon/mandel4ts.git')
# job step2: run mandelbrot merge
job.setExecutable('./mandel4ts/merge_data.py')
outputPath = os.path.join('/vo.france-grilles.fr/user', owner[0], owner,
'mandelbrot/images/merged')
outputPattern = 'data_merged*txt'
outputSE = 'DIRAC-USER'
nb_input_files = 7
outputMetadata = json.dumps({
"application": "mandelbrot",
"image_format": "ascii",
"image_width": 7680,
"image_height": 200 * nb_input_files,
"owner": owner
})
# job step3: upload data and set metadata
job.setExecutable('./mandel4ts/dirac-add-files.py',
arguments="%s '%s' %s '%s'" %
(outputPath, outputPattern, outputSE, outputMetadata))
########################################
# Transformation definition
########################################
t = Transformation()
t.setTransformationName(owner + '_step2')
t.setType("DataReprocessing")
t.setDescription("Merge mandelbrot images production")
t.setLongDescription("Merge mandelbrot images production")
t.setGroupSize(nb_input_files) # group input files
# set the job workflow to the transformation
t.setBody(job.workflow.toXML())
# define input data by metadata query
inputMetaquery = json.dumps({
"application": "mandelbrot",
"image_format": "ascii",
"image_width": 7680,
"image_height": 200,
"owner": owner
})
t.setFileMask(inputMetaquery)
########################################
# Transformation submission
########################################
res = t.addTransformation()
if not res['OK']:
print(res['Message'])
DIRAC.exit(-1)
t.setStatus("Active")
t.setAgentType("Automatic")
return res
def submitTS():
########################################
# Modify here with your dirac username
owner = 'user02'
########################################
########################################
# Job description
########################################
job = Job()
job.setName('build mandelbrot')
job.setOutputSandbox( ['*log'] )
job.setType('DataReprocessing')
## define the job workflow in 3 steps
# job step1: setup software
job.setExecutable('git clone https://github.com/bregeon/mandel4ts.git')
# job step2: run mandelbrot build image
job.setExecutable('./mandel4ts/build_merged_img.py')
outputPath = os.path.join('/vo.france-grilles.fr/user',owner[0],owner,'mandelbrot/images/final')
outputPattern = 'merged_image.bmp'
outputSE = 'DIRAC-USER'
outputMetadata = json.dumps( {"application":"mandelbrot","image_format":"bmp", "image_width":7680, "image_height":4200, "owner":owner} )
# job step3: upload data and set metadata
job.setExecutable( './mandel4ts/dirac-add-files.py', arguments = "%s '%s' %s '%s'" % (outputPath, outputPattern, outputSE, outputMetadata ) )
# job step4: mark input files as done with the FailoverRequest (and a few other things)
job.setExecutable('/bin/ls -l', modulesList=['Script', 'FailoverRequest'])
########################################
# Transformation definition
########################################
t = Transformation()
t.setTransformationName( owner+'_step3' )
t.setType( "DataReprocessing" )
t.setDescription( "Merge mandelbrot images production" )
t.setLongDescription( "Merge mandelbrot images production" )
t.setGroupSize( 3 ) # group input files
# set the job workflow to the transformation
t.setBody ( job.workflow.toXML() )
# define input data by metadata query
inputMetaquery = {"application":"mandelbrot","image_format":"ascii", "image_width":7680, "image_height":1400, "owner":owner}
t.setInputMetaQuery(inputMetaquery)
########################################
# Transformation submission
########################################
res = t.addTransformation()
if not res['OK']:
print(res['Message'])
DIRAC.exit( -1 )
t.setStatus( "Active" )
t.setAgentType( "Automatic" )
return res
def DataReplicaRemovalTSExample(args=None):
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
if (len(args) != 1):
Script.gLogger.notice('Wrong number of arguments')
Script.showHelp()
infile = args[0]
f = open(infile, 'r')
infileList = []
for line in f:
infile = line.strip()
if line != "\n":
infileList.append(infile)
t = Transformation()
tc = TransformationClient()
t.setType("Removal")
t.setPlugin("Broadcast") # Mandatory for ReplicaRemoval
t.setSourceSE(['CC-IN2P3-Tape'
]) # A list of SE where at least 1 SE is the valid one
t.setTargetSE([
'CEA-Disk', 'LAPP-Disk', 'DESY-ZN-Disk', 'CC-IN2P3-Disk',
'CYF-STORM-Disk'
]) # A list of SE where at least 1 SE is the valid one
t.setDescription("AarProd2 ReplicaRemoval")
t.setLongDescription("AarProd2 ReplicaRemoval") # Mandatory
t.setGroupSize(
100
) # Here you specify how many files should be grouped within the same request, e.g. 100
t.setBody("Removal;RemoveReplica"
) # Mandatory (the default is a ReplicateAndRegister operation)
t.addTransformation() # Transformation is created here
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
tc.addFilesToTransformation(transID['Value'],
infileList) # Files are added here
def submitTS(job, transName, mqJson):
""" Create a transformation executing the job workflow
This is using a file mask so that files be added on the fly.
"""
DIRAC.gLogger.notice('submitTS')
# Initialize JOB_ID
job.workflow.addParameter(
Parameter("JOB_ID", "000000", "string", "", "", True, False,
"Temporary fix"))
t = Transformation()
t.setTransformationName(transName) # this must be unique
t.setType("DataReprocessing")
t.setDescription("EvnDisp3MSCW example")
t.setLongDescription("EvnDisplay stereo reconstruction") # mandatory
t.setBody(job.workflow.toXML())
t.setGroupSize(100)
t.setFileMask(mqJson) # catalog query is defined here
t.addTransformation() # transformation is created here
t.setStatus("Active")
t.setAgentType("Automatic")
return
def _createReplication( targetSE, sourceSE, prodID, datatype, extraname=''):
"""Creates the replication transformation based on the given parameters"""
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
metadata = {"Datatype":datatype, "ProdID":prodID}
trans = Transformation()
transName = 'replicate_%s_%s' % ( str(prodID), ",".join(targetSE) )
if extraname:
transName += "_%s" % extraname
trans.setTransformationName( transName )
description = 'Replicate files for prodID %s to %s' % ( str(prodID), ",".join(targetSE) )
trans.setDescription( description )
trans.setLongDescription( description )
trans.setType( 'Replication' )
trans.setPlugin( 'Broadcast' )
res = trans.setSourceSE( sourceSE )
if not res['OK']:
exit(1)
res = trans.setTargetSE( targetSE )
if not res['OK']:
exit(1)
res = trans.addTransformation()
if not res['OK']:
gLogger.error(res['Message'])
exit(1)
gLogger.verbose(res)
trans.setStatus( 'Active' )
trans.setAgentType( 'Automatic' )
currtrans = trans.getTransformationID()['Value']
client = TransformationClient()
res = client.createTransformationInputDataQuery( currtrans, metadata )
if res['OK']:
gLogger.always("Successfully created replication transformation")
return S_OK()
else:
gLogger.error("Failure during replication creation", res['Message'])
return S_ERROR("Failed to create transformation")
