def _createTrafo(): """reads command line parameters, makes check and creates replication transformation""" clip = Params() clip.registerSwitches(Script) registerSwitches(clip, Script) Script.parseCommandLine() if not clip.checkSettings(Script)['OK']: LOG.error("ERROR: Missing settings") return 1 for index, prodID in enumerate(clip.metaValues): datatype = clip.datatype if clip.datatype else ['GEN', 'SIM', 'REC'][index % 3] plugin = 'Broadcast' if clip.forcemoving or clip.flavour != 'Moving' else 'BroadcastProcessed' retData = checkDatatype(prodID, datatype) if not retData['OK']: LOG.error("ERROR: %s" % retData['Message']) return 1 tGroup = getTransformationGroup(prodID, clip.groupName) parDict = dict(flavour='Moving', targetSE=clip.targetSE, sourceSE=clip.sourceSE, metaKey=clip.metaKey, metaValue=prodID, extraData={'Datatype': datatype}, extraname=clip.extraname, plugin=plugin, groupSize=clip.groupSize, tGroup=tGroup, enable=clip.enable, ) LOG.debug("Parameters: %s" % pformat(parDict)) resCreate = createDataTransformation(**parDict) if not resCreate['OK']: return 1 return 0
def _createTrafo(): """reads command line parameters, makes check and creates replication transformation""" clip = Params() clip.registerSwitches(Script) Script.parseCommandLine() if not clip.checkSettings(Script)['OK']: gLogger.error("ERROR: Missing settings") return 1 for metaValue in clip.metaValues: resCreate = createDataTransformation( flavour=clip.flavour, targetSE=clip.targetSE, sourceSE=clip.sourceSE, metaKey=clip.metaKey, metaValue=metaValue, extraData=clip.extraData, extraname=clip.extraname, groupSize=clip.groupSize, plugin=clip.plugin, enable=clip.enable, ) if not resCreate['OK']: gLogger.error("Failed to create Transformation", resCreate['Message']) return 1 return 0
def _createTrafo(): """reads command line parameters, makes check and creates replication transformation""" clip = Params() clip.registerSwitches(Script) registerSwitches(Script) Script.parseCommandLine() if not clip.checkSettings(Script)['OK']: LOG.error("ERROR: Missing settings") return 1 for prodID in clip.metaValues: tGroup = getTransformationGroup(prodID, clip.groupName) parDict = dict(flavour='Replication', targetSE=clip.targetSE, sourceSE=clip.sourceSE, metaKey=clip.metaKey, metaValue=prodID, extraData={'Datatype': clip.datatype}, extraname=clip.extraname, plugin=clip.plugin, groupSize=clip.groupSize, tGroup=tGroup, enable=clip.enable, ) LOG.debug("Parameters: %s" % pformat(parDict)) resCreate = createDataTransformation(**parDict) if not resCreate['OK']: return 1 return 0
def main(): """reads command line parameters, makes check and creates replication transformation""" from DIRAC import gLogger, exit as dexit from DIRAC.TransformationSystem.Utilities.ReplicationCLIParameters import Params clip = Params() clip.registerSwitches(Script) Script.parseCommandLine() from DIRAC.TransformationSystem.Utilities.ReplicationTransformation import createDataTransformation if not clip.checkSettings(Script)["OK"]: gLogger.error("ERROR: Missing settings") dexit(1) for metaValue in clip.metaValues: resCreate = createDataTransformation( flavour=clip.flavour, targetSE=clip.targetSE, sourceSE=clip.sourceSE, metaKey=clip.metaKey, metaValue=metaValue, extraData=clip.extraData, extraname=clip.extraname, groupSize=clip.groupSize, tGroup=clip.groupName, plugin=clip.plugin, enable=clip.enable, ) if not resCreate["OK"]: gLogger.error("Failed to create Transformation", resCreate["Message"]) dexit(1) dexit(0)
def _createTrafo(): """reads command line parameters, makes check and creates replication transformation""" clip = Params() clip.registerSwitches(Script) Script.parseCommandLine() if not clip.checkSettings(Script)['OK']: gLogger.error("ERROR: Missing settings") return 1 for metaValue in clip.metaValues: resCreate = createDataTransformation(flavour=clip.flavour, targetSE=clip.targetSE, sourceSE=clip.sourceSE, metaKey=clip.metaKey, metaValue=metaValue, extraData=clip.extraData, extraname=clip.extraname, groupSize=clip.groupSize, plugin=clip.plugin, enable=clip.enable, ) if not resCreate['OK']: gLogger.error("Failed to create Transformation", resCreate['Message']) return 1 return 0
def test_createRepl_2(self): """ test creating transformation """ tSE = "Target-SRM" sSE = "Source-SRM" prodID = 12345 trmodule = "DIRAC.TransformationSystem.Client.Transformation.Transformation" with patch(trmodule + ".getTransformation", new=Mock(return_value=S_OK({}))), \ patch(trmodule + ".addTransformation", new=Mock(return_value=S_OK())), \ patch(trmodule + "._Transformation__setSE", new=Mock(return_value=S_OK())): ret = createDataTransformation('Moving', tSE, sSE, 'prodID', prodID, extraname="extraName", enable=True) self.assertTrue(ret['OK'], ret.get('Message', ""))
def test_createRepl_createTrafoFail_(self): """ test creating transformation """ tSE = "Target-SRM" sSE = "Source-SRM" prodID = 12345 trmodule = "DIRAC.TransformationSystem.Client.Transformation.Transformation" with patch(trmodule + ".getTransformation", new=Mock(return_value=S_OK({}))), \ patch(trmodule + ".addTransformation", new=Mock(return_value=S_ERROR('Failed to add IMQ'))), \ patch(trmodule + "._Transformation__setSE", new=Mock(return_value=S_OK())): ret = createDataTransformation('Moving', tSE, sSE, 'prodID', prodID, enable=True) self.assertFalse(ret['OK'], str(ret)) self.assertIn("Failed to add IMQ", ret['Message'])
def test_createRepl_NoSource(self): """ test creating transformation """ tSE = 'Target-SRM' sSE = '' prodID = 12345 trmodule = 'DIRAC.TransformationSystem.Client.Transformation.Transformation' with patch(trmodule + '.getTransformation', new=Mock(return_value=S_OK({}))), \ patch(trmodule + '.addTransformation', new=Mock(return_value=S_OK())), \ patch(trmodule + '._Transformation__setSE', new=Mock(return_value=S_OK())): ret = createDataTransformation('Moving', tSE, sSE, 'prodID', prodID, enable=True) self.assertTrue(ret['OK'], ret.get('Message', '')) self.assertEqual(ret['Value'].getPlugin().get('Value'), 'Broadcast')
def test_createRepl_Dry(self): """ test creating transformation """ tSE = "Target-SRM" sSE = "Source-SRM" prodID = 12345 module_name = "DIRAC.TransformationSystem.Utilities.ReplicationTransformation" trmodule = "DIRAC.TransformationSystem.Client.Transformation.Transformation" with patch(trmodule + ".getTransformation", new=Mock(return_value=S_OK({}))), \ patch(trmodule + ".addTransformation", new=Mock(return_value=S_OK())), \ patch(trmodule + "._Transformation__setSE", new=Mock(return_value=S_OK())), \ patch("%s.TransformationClient" % module_name, new=self.tMock): ret = createDataTransformation('Moving', tSE, sSE, 'prodID', prodID, enable=False, extraData={}) self.assertTrue(ret['OK'], ret.get('Message', ""))
def test_createRepl_Broadcast(self): """ test creating transformation """ tSE = "Target-SRM" sSE = "Source-SRM" prodID = 12345 trmodule = "DIRAC.TransformationSystem.Client.Transformation.Transformation" with patch(trmodule + ".getTransformation", new=Mock(return_value=S_OK({}))), \ patch(trmodule + ".addTransformation", new=Mock(return_value=S_OK())), \ patch(trmodule + "._Transformation__setSE", new=Mock(return_value=S_OK())): ret = createDataTransformation('Moving', tSE, sSE, 'prodID', prodID, enable=True) self.assertTrue(ret['OK'], ret.get('Message', "")) self.assertEqual(ret['Value'].getPlugin().get('Value'), 'Broadcast') self.assertEqual(ret['Value'].inputMetaQuery, {'prodID': prodID})
def test_createRepl_2(self): """ test creating transformation """ tSE = "Target-SRM" sSE = "Source-SRM" prodID = 12345 module_name = "DIRAC.TransformationSystem.Utilities.ReplicationTransformation" trmodule = "DIRAC.TransformationSystem.Client.Transformation.Transformation" with patch(trmodule + ".getTransformation", new=Mock(return_value=S_OK({}))), \ patch(trmodule + ".addTransformation", new=Mock(return_value=S_OK())), \ patch(trmodule + "._Transformation__setSE", new=Mock(return_value=S_OK())), \ patch("%s.TransformationClient" % module_name, new=self.tMock): ret = createDataTransformation('Moving', tSE, sSE, 'prodID', prodID, extraname="extraName", enable=True) self.assertTrue(ret['OK'], ret.get('Message', ""))
def test_createRepl_SEFail_1(self): """ test creating transformation """ tSE = "Target-SRM" sSE = "Source-SRM" prodID = 12345 module_name = "DIRAC.TransformationSystem.Utilities.ReplicationTransformation" trmodule = "DIRAC.TransformationSystem.Client.Transformation.Transformation" with patch(trmodule + ".getTransformation", new=Mock(return_value=S_OK({}))), \ patch(trmodule + ".addTransformation", new=Mock(return_value=S_OK())), \ patch(trmodule + "._Transformation__setSE", new=Mock(side_effect=(S_OK(), S_ERROR()))), \ patch("%s.TransformationClient" % module_name, new=self.tMock): ret = createDataTransformation('Moving', tSE, sSE, 'prodID', prodID, enable=True) self.assertFalse(ret['OK'], str(ret)) self.assertIn("TargetSE not valid", ret['Message'])
def test_createRepl_addTrafoFail_(self): """ test creating transformation """ tSE = "Target-SRM" sSE = "Source-SRM" prodID = 12345 module_name = "DIRAC.TransformationSystem.Utilities.ReplicationTransformation" trmodule = "DIRAC.TransformationSystem.Client.Transformation.Transformation" with patch(trmodule + ".getTransformation", new=Mock(return_value=S_OK({}))), \ patch(trmodule + ".addTransformation", new=Mock(return_value=S_ERROR("Cannot add Trafo"))), \ patch(trmodule + "._Transformation__setSE", new=Mock(return_value=S_OK())), \ patch("%s.TransformationClient" % module_name, new=self.tMock): ret = createDataTransformation('Moving', tSE, sSE, 'prodID', prodID, enable=True) self.assertFalse(ret['OK'], str(ret)) self.assertIn("Cannot add Trafo", ret['Message'])
def test_createRepl_BadFlavour(self): """ test creating transformation """ tSE = "Target-SRM" sSE = "Source-SRM" prodID = 12345 module_name = "DIRAC.TransformationSystem.Utilities.ReplicationTransformation" trmodule = "DIRAC.TransformationSystem.Client.Transformation.Transformation" with patch(trmodule + ".getTransformation", new=Mock(return_value=S_OK({}))), \ patch(trmodule + ".addTransformation", new=Mock(return_value=S_OK())), \ patch(trmodule + "._Transformation__setSE", new=Mock(return_value=S_OK())), \ patch("%s.TransformationClient" % module_name, new=self.tMock): ret = createDataTransformation('Smelly', tSE, sSE, 'prodID', prodID, extraname="extraName", enable=True) self.assertFalse(ret['OK'], ret.get('Message', "")) self.assertIn('Unsupported flavour', ret['Message'])
def test_createRepl_createTrafoFail_(self): """ test creating transformation """ tSE = "Target-SRM" sSE = "Source-SRM" prodID = 12345 self.tClientMock.createTransformationInputDataQuery.return_value = S_ERROR("Failed to create IDQ") module_name = "DIRAC.TransformationSystem.Utilities.ReplicationTransformation" trmodule = "DIRAC.TransformationSystem.Client.Transformation.Transformation" with patch(trmodule + ".getTransformation", new=Mock(return_value=S_OK({}))), \ patch(trmodule + ".addTransformation", new=Mock(return_value=S_OK())), \ patch(trmodule + "._Transformation__setSE", new=Mock(return_value=S_OK())), \ patch("%s.TransformationClient" % module_name, new=self.tMock): ret = createDataTransformation('Moving', tSE, sSE, 'prodID', prodID, enable=True) self.assertFalse(ret['OK'], str(ret)) self.assertIn("Failed to create IDQ", ret['Message'])
def createMovingTransformation( self, meta, prodType ): """ create moving transformations for output files """ sourceSE = self.outputSE targetSE = self.finalOutputSE prodID = meta['ProdID'] try: dataTypes = { 'MCReconstruction': ('DST', 'REC'), 'MCReconstruction_Overlay': ('DST', 'REC'), 'MCSimulation': ('SIM',), 'MCGeneration': ('GEN',), }[prodType] except KeyError: raise RuntimeError( "ERROR creating MovingTransformation" + repr(prodType) + "unknown" ) if not any( getattr( self._flags, "move%s" % dataType.capitalize() ) for dataType in dataTypes ): gLogger.notice( "*"*80 + "\nNot creating moving transformation for prodID: %s, %s " % (meta['ProdID'], prodType ) ) return from DIRAC.TransformationSystem.Utilities.ReplicationTransformation import createDataTransformation from DIRAC.TransformationSystem.Client.Transformation import Transformation for dataType in dataTypes: if getattr(self._flags, "move%s" % dataType.capitalize()): gLogger.notice("*" * 80 + "\nCreating moving transformation for prodID: %s, %s, %s " % (meta['ProdID'], prodType, dataType)) parDict = dict(flavour='Moving', targetSE=targetSE, sourceSE=sourceSE, plugin='Broadcast%s' % ('' if dataType.lower() not in ('gen', 'sim') else 'Processed'), metaKey='ProdID', metaValue=prodID, extraData={'Datatype': dataType}, tGroup=self.prodGroup, groupSize=int(self.moveGroupSize), enable=not self._flags.dryRun, ) message = "Moving transformation with parameters" gLogger.notice("%s:\n%s" % (message, pformat(parDict, indent=len(message) + 2, width=120))) res = createDataTransformation(**parDict) if not res['OK']: gLogger.error("Failed to create moving transformation:", res['Message']) elif isinstance(res['Value'], Transformation): newTrans = res['Value'] newTrans.setStatus(self.moveStatus)
def test_createRepl_Standard(self): """ test creating transformation """ tSE = 'Target-SRM' sSE = '' prodID = 12345 module_name = 'DIRAC.TransformationSystem.Utilities.ReplicationTransformation' trmodule = 'DIRAC.TransformationSystem.Client.Transformation.Transformation' with patch(trmodule + '.getTransformation', new=Mock(return_value=S_OK({}))), \ patch(trmodule + '.addTransformation', new=Mock(return_value=S_OK())), \ patch(trmodule + '._Transformation__setSE', new=Mock(return_value=S_OK())), \ patch('%s.TransformationClient' % module_name, new=self.tMock): ret = createDataTransformation('Moving', tSE, sSE, 'prodID', prodID, enable=True) self.assertTrue(ret['OK'], ret.get('Message', '')) self.assertEqual(ret['Value'].getPlugin().get('Value'), 'Standard')
def test_createRepl_SEFail_2(self): """test creating transformation""" tSE = "Target-SRM" sSE = "Source-SRM" prodID = 12345 trmodule = "DIRAC.TransformationSystem.Client.Transformation.Transformation" with patch(trmodule + ".getTransformation", new=Mock(return_value=S_OK({}))), patch( trmodule + ".addTransformation", new=Mock(return_value=S_OK())), patch( trmodule + "._Transformation__setSE", new=Mock(side_effect=(S_ERROR(), S_ERROR()))): ret = createDataTransformation("Moving", tSE, sSE, "prodID", prodID, enable=True) self.assertFalse(ret["OK"], str(ret)) self.assertIn("SourceSE not valid", ret["Message"])
def test_createRepl_NoSource(self): """test creating transformation""" tSE = "Target-SRM" sSE = "" prodID = 12345 trmodule = "DIRAC.TransformationSystem.Client.Transformation.Transformation" with patch(trmodule + ".getTransformation", new=Mock(return_value=S_OK({}))), patch( trmodule + ".addTransformation", new=Mock(return_value=S_OK())), patch( trmodule + "._Transformation__setSE", new=Mock(return_value=S_OK())): ret = createDataTransformation("Moving", tSE, sSE, "prodID", prodID, enable=True) self.assertTrue(ret["OK"], ret.get("Message", "")) self.assertEqual(ret["Value"].getPlugin().get("Value"), "Broadcast")
def test_createRepl_Dry(self): """test creating transformation""" tSE = "Target-SRM" sSE = "Source-SRM" prodID = 12345 trmodule = "DIRAC.TransformationSystem.Client.Transformation.Transformation" with patch(trmodule + ".getTransformation", new=Mock(return_value=S_OK({}))), patch( trmodule + ".addTransformation", new=Mock(return_value=S_OK())), patch( trmodule + "._Transformation__setSE", new=Mock(return_value=S_OK())): ret = createDataTransformation("Moving", tSE, sSE, "prodID", prodID, enable=False, extraData={}) self.assertTrue(ret["OK"], ret.get("Message", ""))
def test_createRepl_BadFlavour(self): """test creating transformation""" tSE = "Target-SRM" sSE = "Source-SRM" prodID = 12345 trmodule = "DIRAC.TransformationSystem.Client.Transformation.Transformation" with patch(trmodule + ".getTransformation", new=Mock(return_value=S_OK({}))), patch( trmodule + ".addTransformation", new=Mock(return_value=S_OK())), patch( trmodule + "._Transformation__setSE", new=Mock(return_value=S_OK())): ret = createDataTransformation("Smelly", tSE, sSE, "prodID", prodID, extraname="extraName", enable=True) self.assertFalse(ret["OK"], ret.get("Message", "")) self.assertIn("Unsupported flavour", ret["Message"])
tape_se = argss[1] tc = TransformationClient() # Check input data set information name, n_files, size, meta_query = get_dataset_info(dataset_name) print('Found dataset %s with %d files.' % (name, n_files)) # choose a metaKey meta_key = 'analysis_prog' meta_value = 'merge_simtel' tag = '' do_it = True se_list = ['CC-IN2P3-Disk', 'DESY-ZN-Disk', 'CYF-STORM-Disk'] # create Transformation data_ts = createDataTransformation( flavour='Moving', targetSE=tape_se, sourceSE=se_list, metaKey=meta_key, metaValue=meta_value, extraData=meta_query, extraname=tag, groupSize=1, plugin='Broadcast', tGroup=None, tBody=None, enable=do_it, )