def _createTrafo():
"""reads command line parameters, makes check and creates replication transformation"""
clip = Params()
clip.registerSwitches(Script)
registerSwitches(clip, Script)
Script.parseCommandLine()
if not clip.checkSettings(Script)['OK']:
LOG.error("ERROR: Missing settings")
return 1
for index, prodID in enumerate(clip.metaValues):
datatype = clip.datatype if clip.datatype else ['GEN', 'SIM', 'REC'][index % 3]
plugin = 'Broadcast' if clip.forcemoving or clip.flavour != 'Moving' else 'BroadcastProcessed'
retData = checkDatatype(prodID, datatype)
if not retData['OK']:
LOG.error("ERROR: %s" % retData['Message'])
return 1
tGroup = getTransformationGroup(prodID, clip.groupName)
parDict = dict(flavour='Moving',
targetSE=clip.targetSE,
sourceSE=clip.sourceSE,
metaKey=clip.metaKey,
metaValue=prodID,
extraData={'Datatype': datatype},
extraname=clip.extraname,
plugin=plugin,
groupSize=clip.groupSize,
tGroup=tGroup,
enable=clip.enable,
)
LOG.debug("Parameters: %s" % pformat(parDict))
resCreate = createDataTransformation(**parDict)
if not resCreate['OK']:
return 1
return 0
def _createTrafo():
"""reads command line parameters, makes check and creates replication transformation"""
clip = Params()
clip.registerSwitches(Script)
Script.parseCommandLine()
if not clip.checkSettings(Script)['OK']:
gLogger.error("ERROR: Missing settings")
return 1
for metaValue in clip.metaValues:
resCreate = createDataTransformation(
flavour=clip.flavour,
targetSE=clip.targetSE,
sourceSE=clip.sourceSE,
metaKey=clip.metaKey,
metaValue=metaValue,
extraData=clip.extraData,
extraname=clip.extraname,
groupSize=clip.groupSize,
plugin=clip.plugin,
enable=clip.enable,
)
if not resCreate['OK']:
gLogger.error("Failed to create Transformation",
resCreate['Message'])
return 1
return 0
def _createTrafo():
"""reads command line parameters, makes check and creates replication transformation"""
clip = Params()
clip.registerSwitches(Script)
registerSwitches(Script)
Script.parseCommandLine()
if not clip.checkSettings(Script)['OK']:
LOG.error("ERROR: Missing settings")
return 1
for prodID in clip.metaValues:
tGroup = getTransformationGroup(prodID, clip.groupName)
parDict = dict(flavour='Replication',
targetSE=clip.targetSE,
sourceSE=clip.sourceSE,
metaKey=clip.metaKey,
metaValue=prodID,
extraData={'Datatype': clip.datatype},
extraname=clip.extraname,
plugin=clip.plugin,
groupSize=clip.groupSize,
tGroup=tGroup,
enable=clip.enable,
)
LOG.debug("Parameters: %s" % pformat(parDict))
resCreate = createDataTransformation(**parDict)
if not resCreate['OK']:
return 1
return 0
def main():
"""reads command line parameters, makes check and creates replication transformation"""
from DIRAC import gLogger, exit as dexit
from DIRAC.TransformationSystem.Utilities.ReplicationCLIParameters import Params
clip = Params()
clip.registerSwitches(Script)
Script.parseCommandLine()
from DIRAC.TransformationSystem.Utilities.ReplicationTransformation import createDataTransformation
if not clip.checkSettings(Script)["OK"]:
gLogger.error("ERROR: Missing settings")
dexit(1)
for metaValue in clip.metaValues:
resCreate = createDataTransformation(
flavour=clip.flavour,
targetSE=clip.targetSE,
sourceSE=clip.sourceSE,
metaKey=clip.metaKey,
metaValue=metaValue,
extraData=clip.extraData,
extraname=clip.extraname,
groupSize=clip.groupSize,
tGroup=clip.groupName,
plugin=clip.plugin,
enable=clip.enable,
)
if not resCreate["OK"]:
gLogger.error("Failed to create Transformation",
resCreate["Message"])
dexit(1)
dexit(0)
def _createTrafo():
"""reads command line parameters, makes check and creates replication transformation"""
clip = Params()
clip.registerSwitches(Script)
Script.parseCommandLine()
if not clip.checkSettings(Script)['OK']:
gLogger.error("ERROR: Missing settings")
return 1
for metaValue in clip.metaValues:
resCreate = createDataTransformation(flavour=clip.flavour,
targetSE=clip.targetSE,
sourceSE=clip.sourceSE,
metaKey=clip.metaKey,
metaValue=metaValue,
extraData=clip.extraData,
extraname=clip.extraname,
groupSize=clip.groupSize,
plugin=clip.plugin,
enable=clip.enable,
)
if not resCreate['OK']:
gLogger.error("Failed to create Transformation", resCreate['Message'])
return 1
return 0
def test_createRepl_2(self):
""" test creating transformation """
tSE = "Target-SRM"
sSE = "Source-SRM"
prodID = 12345
trmodule = "DIRAC.TransformationSystem.Client.Transformation.Transformation"
with patch(trmodule + ".getTransformation", new=Mock(return_value=S_OK({}))), \
patch(trmodule + ".addTransformation", new=Mock(return_value=S_OK())), \
patch(trmodule + "._Transformation__setSE", new=Mock(return_value=S_OK())):
ret = createDataTransformation('Moving', tSE, sSE, 'prodID', prodID, extraname="extraName", enable=True)
self.assertTrue(ret['OK'], ret.get('Message', ""))
def test_createRepl_createTrafoFail_(self):
""" test creating transformation """
tSE = "Target-SRM"
sSE = "Source-SRM"
prodID = 12345
trmodule = "DIRAC.TransformationSystem.Client.Transformation.Transformation"
with patch(trmodule + ".getTransformation", new=Mock(return_value=S_OK({}))), \
patch(trmodule + ".addTransformation", new=Mock(return_value=S_ERROR('Failed to add IMQ'))), \
patch(trmodule + "._Transformation__setSE", new=Mock(return_value=S_OK())):
ret = createDataTransformation('Moving', tSE, sSE, 'prodID', prodID, enable=True)
self.assertFalse(ret['OK'], str(ret))
self.assertIn("Failed to add IMQ", ret['Message'])
def test_createRepl_NoSource(self):
""" test creating transformation """
tSE = 'Target-SRM'
sSE = ''
prodID = 12345
trmodule = 'DIRAC.TransformationSystem.Client.Transformation.Transformation'
with patch(trmodule + '.getTransformation', new=Mock(return_value=S_OK({}))), \
patch(trmodule + '.addTransformation', new=Mock(return_value=S_OK())), \
patch(trmodule + '._Transformation__setSE', new=Mock(return_value=S_OK())):
ret = createDataTransformation('Moving', tSE, sSE, 'prodID', prodID, enable=True)
self.assertTrue(ret['OK'], ret.get('Message', ''))
self.assertEqual(ret['Value'].getPlugin().get('Value'), 'Broadcast')
def test_createRepl_Dry(self):
""" test creating transformation """
tSE = "Target-SRM"
sSE = "Source-SRM"
prodID = 12345
module_name = "DIRAC.TransformationSystem.Utilities.ReplicationTransformation"
trmodule = "DIRAC.TransformationSystem.Client.Transformation.Transformation"
with patch(trmodule + ".getTransformation", new=Mock(return_value=S_OK({}))), \
patch(trmodule + ".addTransformation", new=Mock(return_value=S_OK())), \
patch(trmodule + "._Transformation__setSE", new=Mock(return_value=S_OK())), \
patch("%s.TransformationClient" % module_name, new=self.tMock):
ret = createDataTransformation('Moving', tSE, sSE, 'prodID', prodID, enable=False, extraData={})
self.assertTrue(ret['OK'], ret.get('Message', ""))
def test_createRepl_Broadcast(self):
""" test creating transformation """
tSE = "Target-SRM"
sSE = "Source-SRM"
prodID = 12345
trmodule = "DIRAC.TransformationSystem.Client.Transformation.Transformation"
with patch(trmodule + ".getTransformation", new=Mock(return_value=S_OK({}))), \
patch(trmodule + ".addTransformation", new=Mock(return_value=S_OK())), \
patch(trmodule + "._Transformation__setSE", new=Mock(return_value=S_OK())):
ret = createDataTransformation('Moving', tSE, sSE, 'prodID', prodID, enable=True)
self.assertTrue(ret['OK'], ret.get('Message', ""))
self.assertEqual(ret['Value'].getPlugin().get('Value'), 'Broadcast')
self.assertEqual(ret['Value'].inputMetaQuery, {'prodID': prodID})
def test_createRepl_2(self):
""" test creating transformation """
tSE = "Target-SRM"
sSE = "Source-SRM"
prodID = 12345
module_name = "DIRAC.TransformationSystem.Utilities.ReplicationTransformation"
trmodule = "DIRAC.TransformationSystem.Client.Transformation.Transformation"
with patch(trmodule + ".getTransformation", new=Mock(return_value=S_OK({}))), \
patch(trmodule + ".addTransformation", new=Mock(return_value=S_OK())), \
patch(trmodule + "._Transformation__setSE", new=Mock(return_value=S_OK())), \
patch("%s.TransformationClient" % module_name, new=self.tMock):
ret = createDataTransformation('Moving', tSE, sSE, 'prodID', prodID, extraname="extraName", enable=True)
self.assertTrue(ret['OK'], ret.get('Message', ""))
def test_createRepl_SEFail_1(self):
""" test creating transformation """
tSE = "Target-SRM"
sSE = "Source-SRM"
prodID = 12345
module_name = "DIRAC.TransformationSystem.Utilities.ReplicationTransformation"
trmodule = "DIRAC.TransformationSystem.Client.Transformation.Transformation"
with patch(trmodule + ".getTransformation", new=Mock(return_value=S_OK({}))), \
patch(trmodule + ".addTransformation", new=Mock(return_value=S_OK())), \
patch(trmodule + "._Transformation__setSE", new=Mock(side_effect=(S_OK(), S_ERROR()))), \
patch("%s.TransformationClient" % module_name, new=self.tMock):
ret = createDataTransformation('Moving', tSE, sSE, 'prodID', prodID, enable=True)
self.assertFalse(ret['OK'], str(ret))
self.assertIn("TargetSE not valid", ret['Message'])
def test_createRepl_addTrafoFail_(self):
""" test creating transformation """
tSE = "Target-SRM"
sSE = "Source-SRM"
prodID = 12345
module_name = "DIRAC.TransformationSystem.Utilities.ReplicationTransformation"
trmodule = "DIRAC.TransformationSystem.Client.Transformation.Transformation"
with patch(trmodule + ".getTransformation", new=Mock(return_value=S_OK({}))), \
patch(trmodule + ".addTransformation", new=Mock(return_value=S_ERROR("Cannot add Trafo"))), \
patch(trmodule + "._Transformation__setSE", new=Mock(return_value=S_OK())), \
patch("%s.TransformationClient" % module_name, new=self.tMock):
ret = createDataTransformation('Moving', tSE, sSE, 'prodID', prodID, enable=True)
self.assertFalse(ret['OK'], str(ret))
self.assertIn("Cannot add Trafo", ret['Message'])
def test_createRepl_BadFlavour(self):
""" test creating transformation """
tSE = "Target-SRM"
sSE = "Source-SRM"
prodID = 12345
module_name = "DIRAC.TransformationSystem.Utilities.ReplicationTransformation"
trmodule = "DIRAC.TransformationSystem.Client.Transformation.Transformation"
with patch(trmodule + ".getTransformation", new=Mock(return_value=S_OK({}))), \
patch(trmodule + ".addTransformation", new=Mock(return_value=S_OK())), \
patch(trmodule + "._Transformation__setSE", new=Mock(return_value=S_OK())), \
patch("%s.TransformationClient" % module_name, new=self.tMock):
ret = createDataTransformation('Smelly', tSE, sSE, 'prodID', prodID, extraname="extraName", enable=True)
self.assertFalse(ret['OK'], ret.get('Message', ""))
self.assertIn('Unsupported flavour', ret['Message'])
def test_createRepl_createTrafoFail_(self):
""" test creating transformation """
tSE = "Target-SRM"
sSE = "Source-SRM"
prodID = 12345
self.tClientMock.createTransformationInputDataQuery.return_value = S_ERROR("Failed to create IDQ")
module_name = "DIRAC.TransformationSystem.Utilities.ReplicationTransformation"
trmodule = "DIRAC.TransformationSystem.Client.Transformation.Transformation"
with patch(trmodule + ".getTransformation", new=Mock(return_value=S_OK({}))), \
patch(trmodule + ".addTransformation", new=Mock(return_value=S_OK())), \
patch(trmodule + "._Transformation__setSE", new=Mock(return_value=S_OK())), \
patch("%s.TransformationClient" % module_name, new=self.tMock):
ret = createDataTransformation('Moving', tSE, sSE, 'prodID', prodID, enable=True)
self.assertFalse(ret['OK'], str(ret))
self.assertIn("Failed to create IDQ", ret['Message'])
def createMovingTransformation( self, meta, prodType ):
""" create moving transformations for output files """
sourceSE = self.outputSE
targetSE = self.finalOutputSE
prodID = meta['ProdID']
try:
dataTypes = { 'MCReconstruction': ('DST', 'REC'),
'MCReconstruction_Overlay': ('DST', 'REC'),
'MCSimulation': ('SIM',),
'MCGeneration': ('GEN',),
}[prodType]
except KeyError:
raise RuntimeError( "ERROR creating MovingTransformation" + repr(prodType) + "unknown" )
if not any( getattr( self._flags, "move%s" % dataType.capitalize() ) for dataType in dataTypes ):
gLogger.notice( "*"*80 + "\nNot creating moving transformation for prodID: %s, %s " % (meta['ProdID'], prodType ) )
return
from DIRAC.TransformationSystem.Utilities.ReplicationTransformation import createDataTransformation
from DIRAC.TransformationSystem.Client.Transformation import Transformation
for dataType in dataTypes:
if getattr(self._flags, "move%s" % dataType.capitalize()):
gLogger.notice("*" * 80 + "\nCreating moving transformation for prodID: %s, %s, %s " %
(meta['ProdID'], prodType, dataType))
parDict = dict(flavour='Moving',
targetSE=targetSE,
sourceSE=sourceSE,
plugin='Broadcast%s' % ('' if dataType.lower() not in ('gen', 'sim') else 'Processed'),
metaKey='ProdID',
metaValue=prodID,
extraData={'Datatype': dataType},
tGroup=self.prodGroup,
groupSize=int(self.moveGroupSize),
enable=not self._flags.dryRun,
)
message = "Moving transformation with parameters"
gLogger.notice("%s:\n%s" % (message, pformat(parDict, indent=len(message) + 2, width=120)))
res = createDataTransformation(**parDict)
if not res['OK']:
gLogger.error("Failed to create moving transformation:", res['Message'])
elif isinstance(res['Value'], Transformation):
newTrans = res['Value']
newTrans.setStatus(self.moveStatus)
def test_createRepl_Standard(self):
""" test creating transformation """
tSE = 'Target-SRM'
sSE = ''
prodID = 12345
module_name = 'DIRAC.TransformationSystem.Utilities.ReplicationTransformation'
trmodule = 'DIRAC.TransformationSystem.Client.Transformation.Transformation'
with patch(trmodule + '.getTransformation', new=Mock(return_value=S_OK({}))), \
patch(trmodule + '.addTransformation', new=Mock(return_value=S_OK())), \
patch(trmodule + '._Transformation__setSE', new=Mock(return_value=S_OK())), \
patch('%s.TransformationClient' % module_name, new=self.tMock):
ret = createDataTransformation('Moving',
tSE,
sSE,
'prodID',
prodID,
enable=True)
self.assertTrue(ret['OK'], ret.get('Message', ''))
self.assertEqual(ret['Value'].getPlugin().get('Value'), 'Standard')
def test_createRepl_SEFail_2(self):
"""test creating transformation"""
tSE = "Target-SRM"
sSE = "Source-SRM"
prodID = 12345
trmodule = "DIRAC.TransformationSystem.Client.Transformation.Transformation"
with patch(trmodule + ".getTransformation",
new=Mock(return_value=S_OK({}))), patch(
trmodule + ".addTransformation",
new=Mock(return_value=S_OK())), patch(
trmodule + "._Transformation__setSE",
new=Mock(side_effect=(S_ERROR(), S_ERROR()))):
ret = createDataTransformation("Moving",
tSE,
sSE,
"prodID",
prodID,
enable=True)
self.assertFalse(ret["OK"], str(ret))
self.assertIn("SourceSE not valid", ret["Message"])
def test_createRepl_NoSource(self):
"""test creating transformation"""
tSE = "Target-SRM"
sSE = ""
prodID = 12345
trmodule = "DIRAC.TransformationSystem.Client.Transformation.Transformation"
with patch(trmodule + ".getTransformation",
new=Mock(return_value=S_OK({}))), patch(
trmodule + ".addTransformation",
new=Mock(return_value=S_OK())), patch(
trmodule + "._Transformation__setSE",
new=Mock(return_value=S_OK())):
ret = createDataTransformation("Moving",
tSE,
sSE,
"prodID",
prodID,
enable=True)
self.assertTrue(ret["OK"], ret.get("Message", ""))
self.assertEqual(ret["Value"].getPlugin().get("Value"), "Broadcast")
def test_createRepl_Dry(self):
"""test creating transformation"""
tSE = "Target-SRM"
sSE = "Source-SRM"
prodID = 12345
trmodule = "DIRAC.TransformationSystem.Client.Transformation.Transformation"
with patch(trmodule + ".getTransformation",
new=Mock(return_value=S_OK({}))), patch(
trmodule + ".addTransformation",
new=Mock(return_value=S_OK())), patch(
trmodule + "._Transformation__setSE",
new=Mock(return_value=S_OK())):
ret = createDataTransformation("Moving",
tSE,
sSE,
"prodID",
prodID,
enable=False,
extraData={})
self.assertTrue(ret["OK"], ret.get("Message", ""))
def test_createRepl_BadFlavour(self):
"""test creating transformation"""
tSE = "Target-SRM"
sSE = "Source-SRM"
prodID = 12345
trmodule = "DIRAC.TransformationSystem.Client.Transformation.Transformation"
with patch(trmodule + ".getTransformation",
new=Mock(return_value=S_OK({}))), patch(
trmodule + ".addTransformation",
new=Mock(return_value=S_OK())), patch(
trmodule + "._Transformation__setSE",
new=Mock(return_value=S_OK())):
ret = createDataTransformation("Smelly",
tSE,
sSE,
"prodID",
prodID,
extraname="extraName",
enable=True)
self.assertFalse(ret["OK"], ret.get("Message", ""))
self.assertIn("Unsupported flavour", ret["Message"])
tape_se = argss[1]
tc = TransformationClient()
# Check input data set information
name, n_files, size, meta_query = get_dataset_info(dataset_name)
print('Found dataset %s with %d files.' % (name, n_files))
# choose a metaKey
meta_key = 'analysis_prog'
meta_value = 'merge_simtel'
tag = ''
do_it = True
se_list = ['CC-IN2P3-Disk', 'DESY-ZN-Disk', 'CYF-STORM-Disk']
# create Transformation
data_ts = createDataTransformation(
flavour='Moving',
targetSE=tape_se,
sourceSE=se_list,
metaKey=meta_key,
metaValue=meta_value,
extraData=meta_query,
extraname=tag,
groupSize=1,
plugin='Broadcast',
tGroup=None,
tBody=None,
enable=do_it,
)