def test_createFile_NoOutputSet(self):
from ILCDIRAC.Core.Utilities.PrepareOptionFiles import prepareXMLFile
res = prepareXMLFile(
finalxml="marlinNoOutput.xml",
inputXML="marlinInputSmall.xml",
inputGEAR="gearMyFile.xml",
inputSLCIO="mySLCIOInput.slcio",
numberofevts=501,
outputFile='',
outputREC='',
outputDST='',
debug=True,
dd4hepGeoFile="/cvmfs/monty.python.fr/myDetector.xml")
self.assertTrue(res['OK'], res.get('Message'))
self.testedTree = TestPrepareMarlinXMLFile.getTree(
"marlinNoOutput.xml")
#self.assertTrue( self.findProcessorInTree( "BGoverLay" ), "Problem with InitDD4hep" )
self.assertTrue(
*self.checkGlobalTag("LCIOInputFiles", "mySLCIOInput.slcio"))
self.assertTrue(*self.checkGlobalTag("MaxRecordNumber", 501))
self.assertTrue(*self.checkGlobalTag("SkipNEvents", 0))
self.assertTrue(*self.checkGlobalTag("GearXMLFile", "gearMyFile.xml"))
self.assertTrue(*self.checkGlobalTag("Verbosity", "WARNING"))
self.assertTrue(
self.checkProcessorParameter("MyLCIOOutputProcessor",
"LCIOOutputFile", ""))
def test_createFile_productionOutputFiles( self ):
from ILCDIRAC.Core.Utilities.PrepareOptionFiles import prepareXMLFile
res = prepareXMLFile( finalxml="outputprod.xml",
inputXML="marlinRecoProd.xml",
inputGEAR="gearMyFile.xml",
inputSLCIO="mySLCIOInput.slcio",
numberofevts=501,
outputFile= '',
outputREC="outputrec.slcio",
outputDST="outputdst.slcio",
debug=True,
dd4hepGeoFile="/cvmfs/monty.python.fr/myDetector.xml",
overlayParam=[ ('gghad', 0, None) ],
)
self.assertTrue( res['OK'], res.get('Message') )
self.testedTree = TestPrepareMarlinXMLFile.getTree( "outputprod.xml" )
self.assertTrue( self.findProcessorInTree( "InitDD4hep" ),
"Problem with InitDD4hep" )
self.assertTrue( *self.checkGlobalTag( "LCIOInputFiles", "mySLCIOInput.slcio" ) )
self.assertTrue( *self.checkGlobalTag( "MaxRecordNumber", 501 ) )
self.assertTrue( *self.checkGlobalTag( "SkipNEvents", 0 ) )
self.assertTrue( *self.checkGlobalTag( "GearXMLFile", "gearMyFile.xml" ) )
self.assertTrue( *self.checkGlobalTag( "Verbosity", "WARNING" ) )
self.assertTrue( self.checkProcessorParameter(
"InitDD4hep", "DD4hepXMLFile", "/cvmfs/monty.python.fr/myDetector.xml") )
self.assertTrue( self.checkProcessorParameter(
"MyOverlayTiming", "BackgroundFileNames", "file1\nfile2") )
self.assertTrue( self.checkProcessorParameter(
"MyLCIOOutputProcessor", "LCIOOutputFile", "outputrec.slcio") )
self.assertTrue( self.checkProcessorParameter(
"DSTOutput", "LCIOOutputFile", "outputdst.slcio") )
def test_createFile_ildRecoFile( self ):
from ILCDIRAC.Core.Utilities.PrepareOptionFiles import prepareXMLFile
res = prepareXMLFile( finalxml="outputprodild.xml",
inputXML="marlininputild.xml",
inputGEAR="gearMyFile.xml",
inputSLCIO="mySLCIOInput.slcio",
numberofevts=501,
outputFile= '',
outputREC="outputrec.slcio",
outputDST="outputdst.slcio",
debug=True,
dd4hepGeoFile="/cvmfs/monty.python.fr/myDetector.xml",
overlayParam=[ ('aa_lowpt', 333, None) ],
)
self.assertTrue( res['OK'], res.get('Message') )
self.testedTree = TestPrepareMarlinXMLFile.getTree( "outputprodild.xml" )
#self.assertTrue( self.findProcessorInTree( "BGoverLay" ), "Problem with InitDD4hep" )
self.assertTrue( *self.checkGlobalTag( "LCIOInputFiles", "mySLCIOInput.slcio" ) )
self.assertTrue( *self.checkGlobalTag( "MaxRecordNumber", 501 ) )
self.assertTrue( *self.checkGlobalTag( "SkipNEvents", 0 ) )
self.assertTrue( *self.checkGlobalTag( "GearXMLFile", "gearMyFile.xml" ) )
self.assertTrue( *self.checkGlobalTag( "Verbosity", "WARNING" ) )
self.assertTrue( self.checkProcessorParameter( "BgOverlay", "InputFileNames",
"file1\nfile2") )
self.assertTrue( self.checkProcessorParameter( "BgOverlay", "NSkipEventsRandom",
"666") )
def test_createFile_ildRecoFile(self):
from ILCDIRAC.Core.Utilities.PrepareOptionFiles import prepareXMLFile
res = prepareXMLFile(
finalxml="outputprodild.xml",
inputXML="marlininputild.xml",
inputGEAR="gearMyFile.xml",
inputSLCIO="mySLCIOInput.slcio",
numberofevts=501,
outputFile='',
outputREC="outputrec.slcio",
outputDST="outputdst.slcio",
debug=True,
dd4hepGeoFile="/cvmfs/monty.python.fr/myDetector.xml",
overlayParam=[('aa_lowpt', 333, None)],
)
self.assertTrue(res['OK'], res.get('Message'))
self.testedTree = TestPrepareMarlinXMLFile.getTree("outputprodild.xml")
#self.assertTrue( self.findProcessorInTree( "BGoverLay" ), "Problem with InitDD4hep" )
self.assertTrue(
*self.checkGlobalTag("LCIOInputFiles", "mySLCIOInput.slcio"))
self.assertTrue(*self.checkGlobalTag("MaxRecordNumber", 501))
self.assertTrue(*self.checkGlobalTag("SkipNEvents", 0))
self.assertTrue(*self.checkGlobalTag("GearXMLFile", "gearMyFile.xml"))
self.assertTrue(*self.checkGlobalTag("Verbosity", "WARNING"))
self.assertTrue(
self.checkProcessorParameter("BgOverlay", "InputFileNames",
"file1\nfile2"))
self.assertTrue(
self.checkProcessorParameter("BgOverlay", "NSkipEventsRandom",
"666"))
def test_createFile_clicProd2017( self ):
from ILCDIRAC.Core.Utilities.PrepareOptionFiles import prepareXMLFile
res = prepareXMLFile( finalxml="outputprod.xml",
inputXML="clicReconstruction.xml",
inputGEAR="",
inputSLCIO="mySLCIOInput.slcio",
numberofevts=501,
outputFile= '',
outputREC="outputrec.slcio",
outputDST="outputdst.slcio",
debug=False,
dd4hepGeoFile="/cvmfs/monty.python.fr/myDetector.xml",
overlayParam=[ ( 'gghad', 0, 'Overlay380GeV' ) ],
)
self.assertTrue( res['OK'], res.get('Message') )
self.testedTree = TestPrepareMarlinXMLFile.getTree( "outputprod.xml" )
self.assertTrue( *self.checkGlobalTag( "LCIOInputFiles", "mySLCIOInput.slcio" ) )
self.assertTrue( *self.checkGlobalTag( "MaxRecordNumber", 501 ) )
self.assertTrue( *self.checkGlobalTag( "SkipNEvents", 0 ) )
self.assertTrue( *self.checkGlobalTag( "Verbosity", "SILENT" ) )
self.assertTrue( self.checkProcessorParameter(
"InitDD4hep", "DD4hepXMLFile", "/cvmfs/monty.python.fr/myDetector.xml") )
self.assertTrue( self.checkProcessorParameter(
"Overlay380GeV", "BackgroundFileNames", "file1\nfile2") )
self.assertTrue( self.checkProcessorParameter(
"Output_REC", "LCIOOutputFile", "outputrec.slcio") )
self.assertTrue( self.checkProcessorParameter(
"Output_DST", "LCIOOutputFile", "outputdst.slcio") )
def test_createFile( self ):
from ILCDIRAC.Core.Utilities.PrepareOptionFiles import prepareXMLFile
res = prepareXMLFile( finalxml="outputfile.xml",
inputXML="marlininput.xml",
inputGEAR="gearMyFile.xml",
inputSLCIO="mySLCIOInput.slcio",
numberofevts=501,
outputFile= "mySLCIOOutput.slcio",
outputREC="outputrec.slcio",
outputDST="outputdst.slcio",
debug=False,
dd4hepGeoFile="/cvmfs/monty.python.fr/myDetector.xml",
overlayParam=[ ('gghad', 0, None) ],
)
self.assertTrue( res['OK'] )
self.testedTree = TestPrepareMarlinXMLFile.getTree( "outputfile.xml" )
self.assertTrue( self.findProcessorInTree( "InitDD4hep" ), "Problem with InitDD4hep" )
self.assertTrue( *self.checkGlobalTag( "LCIOInputFiles", "mySLCIOInput.slcio" ) )
self.assertTrue( *self.checkGlobalTag( "MaxRecordNumber", 501 ) )
self.assertTrue( *self.checkGlobalTag( "SkipNEvents", 0 ) )
self.assertTrue( *self.checkGlobalTag( "GearXMLFile", "gearMyFile.xml" ) )
self.assertTrue( *self.checkGlobalTag( "Verbosity", "SILENT" ) )
self.assertTrue( self.checkProcessorParameter(
"InitDD4hep", "DD4hepXMLFile", "/cvmfs/monty.python.fr/myDetector.xml") )
self.assertTrue( self.checkProcessorParameter( "MyOverlayTiming",
"BackgroundFileNames",
"file1\nfile2") )
## Make sure these checks are not always true
self.assertFalse( *self.checkGlobalTag( "Verbosity", "NotSILENT" ) )
self.assertFalse( self.checkProcessorParameter( "MyOverlayTiming",
"BackgroundFileNames",
"NotTheseFiles") )
def test_createFile_inputFaulty( self ):
from ILCDIRAC.Core.Utilities.PrepareOptionFiles import prepareXMLFile
res = prepareXMLFile( finalxml="outputprod.xml",
inputXML="marlininputild.xml",
inputGEAR="gearMyFile.xml",
inputSLCIO=None,
numberofevts=501,
outputFile= '',
outputREC="outputrec.slcio",
outputDST="outputdst.slcio",
debug=True,
dd4hepGeoFile="/cvmfs/monty.python.fr/myDetector.xml")
self.assertFalse( res['OK'], res['Message'] )
self.assertIn( "inputSLCIO is neither", res['Message'] )
def test_createFile_overlayTiming_NoFiles( self ):
from ILCDIRAC.Core.Utilities.PrepareOptionFiles import prepareXMLFile
res = prepareXMLFile( finalxml="outputprod.xml",
inputXML="marlinRecoProd.xml",
inputGEAR="gearMyFile.xml",
inputSLCIO="mySLCIOInput.slcio",
numberofevts=501,
outputFile= '',
outputREC="outputrec.slcio",
outputDST="outputdst.slcio",
debug=True,
dd4hepGeoFile="/cvmfs/monty.python.fr/myDetector.xml")
self.assertFalse( res['OK'] )
self.assertIn( "Could not find any overlay files", res['Message'] )
def test_createFile_inputFaulty(self):
from ILCDIRAC.Core.Utilities.PrepareOptionFiles import prepareXMLFile
res = prepareXMLFile(
finalxml="outputprod.xml",
inputXML="marlininputild.xml",
inputGEAR="gearMyFile.xml",
inputSLCIO=None,
numberofevts=501,
outputFile='',
outputREC="outputrec.slcio",
outputDST="outputdst.slcio",
debug=True,
dd4hepGeoFile="/cvmfs/monty.python.fr/myDetector.xml")
self.assertFalse(res['OK'], res['Message'])
self.assertIn("inputSLCIO is neither", res['Message'])
def test_createFile_clicProd2017_noOverlayFiles( self ):
from ILCDIRAC.Core.Utilities.PrepareOptionFiles import prepareXMLFile
res = prepareXMLFile( finalxml="outputprod.xml",
inputXML="clicReconstruction.xml",
inputGEAR="",
inputSLCIO="mySLCIOInput.slcio",
numberofevts=501,
outputFile= '',
outputREC="outputrec.slcio",
outputDST="outputdst.slcio",
debug=False,
dd4hepGeoFile="/cvmfs/monty.python.fr/myDetector.xml",
overlayParam=[ ( 'gghad', 0, 'Overlay380GeV' ) ],
)
assertDiracFailsWith( res, 'Could not find any overlay Files', self )
def test_createFile_bgoverlay_failOverlayActive( self ):
from ILCDIRAC.Core.Utilities.PrepareOptionFiles import prepareXMLFile
res = prepareXMLFile( finalxml="outputprod.xml",
inputXML="marlininputild.xml",
inputGEAR="gearMyFile.xml",
inputSLCIO="mySLCIOInput.slcio",
numberofevts=501,
outputFile= '',
outputREC="outputrec.slcio",
outputDST="outputdst.slcio",
debug=True,
dd4hepGeoFile="/cvmfs/monty.python.fr/myDetector.xml",
overlayParam=[],
)
assertDiracFailsWith( res, 'Found active overlay processors', self )
def test_createFile_bgoverlay_failOverlayActive(self):
from ILCDIRAC.Core.Utilities.PrepareOptionFiles import prepareXMLFile
res = prepareXMLFile(
finalxml="outputprod.xml",
inputXML="marlininputild.xml",
inputGEAR="gearMyFile.xml",
inputSLCIO="mySLCIOInput.slcio",
numberofevts=501,
outputFile='',
outputREC="outputrec.slcio",
outputDST="outputdst.slcio",
debug=True,
dd4hepGeoFile="/cvmfs/monty.python.fr/myDetector.xml",
overlayParam=[],
)
assertDiracFailsWith(res, 'Found active overlay processors', self)
def test_createFile_clicProd2017_noOverlayFiles(self):
from ILCDIRAC.Core.Utilities.PrepareOptionFiles import prepareXMLFile
res = prepareXMLFile(
finalxml="outputprod.xml",
inputXML="clicReconstruction.xml",
inputGEAR="",
inputSLCIO="mySLCIOInput.slcio",
numberofevts=501,
outputFile='',
outputREC="outputrec.slcio",
outputDST="outputdst.slcio",
debug=False,
dd4hepGeoFile="/cvmfs/monty.python.fr/myDetector.xml",
overlayParam=[('gghad', 0, 'Overlay380GeV')],
)
assertDiracFailsWith(res, 'Could not find any overlay Files', self)
def test_createFile_bgoverlay_NoFiles(self):
from ILCDIRAC.Core.Utilities.PrepareOptionFiles import prepareXMLFile
res = prepareXMLFile(
finalxml="outputprod.xml",
inputXML="marlininputild.xml",
inputGEAR="gearMyFile.xml",
inputSLCIO="mySLCIOInput.slcio",
numberofevts=501,
outputFile='',
outputREC="outputrec.slcio",
outputDST="outputdst.slcio",
debug=True,
dd4hepGeoFile="/cvmfs/monty.python.fr/myDetector.xml",
overlayParam=[('aa_lowpt', 0, None)],
)
self.assertFalse(res['OK'])
self.assertIn("Could not find any overlay files", res['Message'])
def test_createFile_inputList( self ):
from ILCDIRAC.Core.Utilities.PrepareOptionFiles import prepareXMLFile
res = prepareXMLFile( finalxml="outputfile.xml",
inputXML="marlininput.xml",
inputGEAR="gearMyFile.xml",
inputSLCIO=["file1","file2"],
numberofevts=501,
outputFile= "mySLCIOOutput.slcio",
outputREC="outputrec.slcio",
outputDST="outputdst.slcio",
debug=False,
dd4hepGeoFile="/cvmfs/monty.python.fr/myDetector.xml")
self.assertTrue( res['OK'] )
self.testedTree = TestPrepareMarlinXMLFile.getTree( "outputfile.xml" )
self.assertTrue( self.findProcessorInTree( "InitDD4hep" ), "Problem with InitDD4hep" )
self.assertTrue( *self.checkGlobalTag( "LCIOInputFiles", "file1 file2" ) )
self.assertTrue( *self.checkGlobalTag( "MaxRecordNumber", 501 ) )
self.assertTrue( *self.checkGlobalTag( "SkipNEvents", 0 ) )
self.assertTrue( *self.checkGlobalTag( "GearXMLFile", "gearMyFile.xml" ) )
self.assertTrue( *self.checkGlobalTag( "Verbosity", "SILENT" ) )
def test_createFile_parseError( self ):
def parseModified( *_args, **_kwargs ):
""" throw exception for parse """
raise RuntimeError("Parse Failed")
from ILCDIRAC.Core.Utilities.PrepareOptionFiles import prepareXMLFile
with patch("%s.ElementTree.parse" % MODULE_NAME, new=parseModified):
res = prepareXMLFile( finalxml="outputprod.xml",
inputXML="marlininputild.xml",
inputGEAR="gearMyFile.xml",
inputSLCIO="mySLCIOInput.slcio",
numberofevts=501,
outputFile= '',
outputREC="outputrec.slcio",
outputDST="outputdst.slcio",
debug=True,
dd4hepGeoFile="/cvmfs/monty.python.fr/myDetector.xml")
self.assertFalse( res['OK'] )
self.assertIn( "Found Exception when parsing", res['Message'] )
def test_createFile_parseError(self):
def parseModified(*_args, **_kwargs):
""" throw exception for parse """
raise RuntimeError("Parse Failed")
from ILCDIRAC.Core.Utilities.PrepareOptionFiles import prepareXMLFile
with patch("%s.ElementTree.parse" % MODULE_NAME, new=parseModified):
res = prepareXMLFile(
finalxml="outputprod.xml",
inputXML="marlininputild.xml",
inputGEAR="gearMyFile.xml",
inputSLCIO="mySLCIOInput.slcio",
numberofevts=501,
outputFile='',
outputREC="outputrec.slcio",
outputDST="outputdst.slcio",
debug=True,
dd4hepGeoFile="/cvmfs/monty.python.fr/myDetector.xml")
self.assertFalse(res['OK'])
self.assertIn("Found Exception when parsing", res['Message'])
def test_createFile_initialParametersMissing(self):
from ILCDIRAC.Core.Utilities.PrepareOptionFiles import prepareXMLFile
res = prepareXMLFile(
finalxml="outputfilesmall.xml",
inputXML="marlinInputSmall.xml",
inputGEAR="gearMyFile.xml",
inputSLCIO="mySLCIOInput.slcio",
numberofevts=501,
outputFile="mySLCIOOutput.slcio",
outputREC="outputrec.slcio",
outputDST="outputdst.slcio",
debug=True,
dd4hepGeoFile="/cvmfs/monty.python.fr/myDetector.xml")
self.assertTrue(res['OK'], res.get('Message'))
self.testedTree = TestPrepareMarlinXMLFile.getTree(
"outputfilesmall.xml")
self.assertTrue(self.findProcessorInTree("InitDD4hep"),
"Problem with InitDD4hep")
self.assertTrue(
*self.checkGlobalTag("LCIOInputFiles", "mySLCIOInput.slcio"))
self.assertTrue(*self.checkGlobalTag("MaxRecordNumber", 501))
self.assertTrue(*self.checkGlobalTag("SkipNEvents", 0))
self.assertTrue(*self.checkGlobalTag("GearXMLFile", "gearMyFile.xml"))
self.assertTrue(*self.checkGlobalTag("Verbosity", "WARNING"))
self.assertTrue(
self.checkProcessorParameter(
"InitDD4hep", "DD4hepXMLFile",
"/cvmfs/monty.python.fr/myDetector.xml"))
self.assertTrue(
self.checkProcessorParameter("MyOverlayTiming", "NBunchtrain",
"0"))
self.assertTrue(
self.checkProcessorParameter("MyOverlayTiming", "NumberBackground",
"0.0"))
self.assertTrue(
self.checkProcessorParameter("MyOverlayTiming",
"BackgroundFileNames", ""))
self.assertTrue(
self.checkProcessorParameter("MyLCIOOutputProcessor",
"LCIOOutputFile",
"mySLCIOOutput.slcio"))
def test_createFile_NoOutputSet( self ):
from ILCDIRAC.Core.Utilities.PrepareOptionFiles import prepareXMLFile
res = prepareXMLFile( finalxml="marlinNoOutput.xml",
inputXML="marlinInputSmall.xml",
inputGEAR="gearMyFile.xml",
inputSLCIO="mySLCIOInput.slcio",
numberofevts=501,
outputFile= '',
outputREC='',
outputDST='',
debug=True,
dd4hepGeoFile="/cvmfs/monty.python.fr/myDetector.xml")
self.assertTrue( res['OK'], res.get('Message') )
self.testedTree = TestPrepareMarlinXMLFile.getTree( "marlinNoOutput.xml" )
#self.assertTrue( self.findProcessorInTree( "BGoverLay" ), "Problem with InitDD4hep" )
self.assertTrue( *self.checkGlobalTag( "LCIOInputFiles", "mySLCIOInput.slcio" ) )
self.assertTrue( *self.checkGlobalTag( "MaxRecordNumber", 501 ) )
self.assertTrue( *self.checkGlobalTag( "SkipNEvents", 0 ) )
self.assertTrue( *self.checkGlobalTag( "GearXMLFile", "gearMyFile.xml" ) )
self.assertTrue( *self.checkGlobalTag( "Verbosity", "WARNING" ) )
self.assertTrue( self.checkProcessorParameter( "MyLCIOOutputProcessor", "LCIOOutputFile", "") )
def test_createFile(self):
from ILCDIRAC.Core.Utilities.PrepareOptionFiles import prepareXMLFile
res = prepareXMLFile(
finalxml="outputfile.xml",
inputXML="marlininput.xml",
inputGEAR="gearMyFile.xml",
inputSLCIO="mySLCIOInput.slcio",
numberofevts=501,
outputFile="mySLCIOOutput.slcio",
outputREC="outputrec.slcio",
outputDST="outputdst.slcio",
debug=False,
dd4hepGeoFile="/cvmfs/monty.python.fr/myDetector.xml",
overlayParam=[('gghad', 0, None)],
)
self.assertTrue(res['OK'])
self.testedTree = TestPrepareMarlinXMLFile.getTree("outputfile.xml")
self.assertTrue(self.findProcessorInTree("InitDD4hep"),
"Problem with InitDD4hep")
self.assertTrue(
*self.checkGlobalTag("LCIOInputFiles", "mySLCIOInput.slcio"))
self.assertTrue(*self.checkGlobalTag("MaxRecordNumber", 501))
self.assertTrue(*self.checkGlobalTag("SkipNEvents", 0))
self.assertTrue(*self.checkGlobalTag("GearXMLFile", "gearMyFile.xml"))
self.assertTrue(*self.checkGlobalTag("Verbosity", "SILENT"))
self.assertTrue(
self.checkProcessorParameter(
"InitDD4hep", "DD4hepXMLFile",
"/cvmfs/monty.python.fr/myDetector.xml"))
self.assertTrue(
self.checkProcessorParameter("MyOverlayTiming",
"BackgroundFileNames",
"file1\nfile2"))
## Make sure these checks are not always true
self.assertFalse(*self.checkGlobalTag("Verbosity", "NotSILENT"))
self.assertFalse(
self.checkProcessorParameter("MyOverlayTiming",
"BackgroundFileNames",
"NotTheseFiles"))
def test_createFile_inputList(self):
from ILCDIRAC.Core.Utilities.PrepareOptionFiles import prepareXMLFile
res = prepareXMLFile(
finalxml="outputfile.xml",
inputXML="marlininput.xml",
inputGEAR="gearMyFile.xml",
inputSLCIO=["file1", "file2"],
numberofevts=501,
outputFile="mySLCIOOutput.slcio",
outputREC="outputrec.slcio",
outputDST="outputdst.slcio",
debug=False,
dd4hepGeoFile="/cvmfs/monty.python.fr/myDetector.xml",
overlayParam=[('gghad', 0, None)])
self.assertTrue(res['OK'], res.get('Message', ''))
self.testedTree = TestPrepareMarlinXMLFile.getTree("outputfile.xml")
self.assertTrue(self.findProcessorInTree("InitDD4hep"),
"Problem with InitDD4hep")
self.assertTrue(*self.checkGlobalTag("LCIOInputFiles", "file1 file2"))
self.assertTrue(*self.checkGlobalTag("MaxRecordNumber", 501))
self.assertTrue(*self.checkGlobalTag("SkipNEvents", 0))
self.assertTrue(*self.checkGlobalTag("GearXMLFile", "gearMyFile.xml"))
self.assertTrue(*self.checkGlobalTag("Verbosity", "SILENT"))
def test_createFile_clicProd2017(self):
from ILCDIRAC.Core.Utilities.PrepareOptionFiles import prepareXMLFile
res = prepareXMLFile(
finalxml="outputprod.xml",
inputXML="clicReconstruction.xml",
inputGEAR="",
inputSLCIO="mySLCIOInput.slcio",
numberofevts=501,
outputFile='',
outputREC="outputrec.slcio",
outputDST="outputdst.slcio",
debug=False,
dd4hepGeoFile="/cvmfs/monty.python.fr/myDetector.xml",
overlayParam=[('gghad', 0, 'Overlay380GeV')],
)
self.assertTrue(res['OK'], res.get('Message'))
self.testedTree = TestPrepareMarlinXMLFile.getTree("outputprod.xml")
self.assertTrue(
*self.checkGlobalTag("LCIOInputFiles", "mySLCIOInput.slcio"))
self.assertTrue(*self.checkGlobalTag("MaxRecordNumber", 501))
self.assertTrue(*self.checkGlobalTag("SkipNEvents", 0))
self.assertTrue(*self.checkGlobalTag("Verbosity", "SILENT"))
self.assertTrue(
self.checkProcessorParameter(
"InitDD4hep", "DD4hepXMLFile",
"/cvmfs/monty.python.fr/myDetector.xml"))
self.assertTrue(
self.checkProcessorParameter("Overlay380GeV",
"BackgroundFileNames",
"file1\nfile2"))
self.assertTrue(
self.checkProcessorParameter("Output_REC", "LCIOOutputFile",
"outputrec.slcio"))
self.assertTrue(
self.checkProcessorParameter("Output_DST", "LCIOOutputFile",
"outputdst.slcio"))
class MarlinAnalysis(DD4hepMixin, ModuleBase):
"""Define the Marlin analysis part of the workflow
"""
def __init__(self):
super(MarlinAnalysis, self).__init__( )
self.enable = True
self.STEP_NUMBER = ''
self.log = gLogger.getSubLogger( "MarlinAnalysis" )
self.result = S_ERROR()
self.inputGEAR = ''
self.outputREC = ''
self.outputDST = ''
self.applicationName = "Marlin"
self.eventstring = ['ProgressHandler','event', 'EventNumber', 'StatusMonitor']
self.envdict = {}
self.ProcessorListToUse = []
self.ProcessorListToExclude = []
self.detectorModel = None
def applicationSpecificInputs(self):
""" Resolve all input variables for the module here.
:return: S_OK()
"""
if 'ParametricInputSandbox' in self.workflow_commons:
paramsb = self.workflow_commons['ParametricInputSandbox']
if not isinstance( paramsb, list ):
if paramsb:
paramsb = paramsb.split(";")
else:
paramsb = []
self.InputFile += paramsb
##Backward compat needed, cannot remove yet.
self.outputREC = self.step_commons.get('outputREC', self.outputREC)
self.outputDST = self.step_commons.get('outputDST', self.outputDST)
if 'IS_PROD' in self.workflow_commons:
if self.workflow_commons["IS_PROD"] and not self.OutputFile:
#self.outputREC = getProdFilename(self.outputREC,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
#self.outputDST = getProdFilename(self.outputDST,int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
#if 'MokkaOutput' in self.workflow_commons:
# self.InputFile = getProdFilename(self.workflow_commons["MokkaOutput"],int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
if 'ProductionOutputData' in self.workflow_commons:
outputlist = self.workflow_commons['ProductionOutputData'].split(";")
for obj in outputlist:
if obj.lower().count("_rec_"):
self.outputREC = os.path.basename(obj)
elif obj.lower().count("_dst_"):
self.outputDST = os.path.basename(obj)
elif obj.lower().count("_sim_"):
self.InputFile = [os.path.basename(obj)]
else:
self.outputREC = getProdFilename(self.outputREC, int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
self.outputDST = getProdFilename(self.outputDST, int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
#if 'MokkaOutput' in self.workflow_commons:
# self.InputFile = getProdFilename(self.workflow_commons["MokkaOutput"],int(self.workflow_commons["PRODUCTION_ID"]),
# int(self.workflow_commons["JOB_ID"]))
self.InputFile = [getProdFilename(self.InputFile, int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))]
if not self.InputFile and self.InputData:
for files in self.InputData:
if files.lower().find(".slcio") > -1:
self.InputFile.append(files)
return S_OK('Parameters resolved')
def runIt(self):
"""
Called by Agent
Execute the following:
- resolve where the soft was installed
- prepare the list of file to feed Marlin with
- create the XML file on which Marlin has to run, done by :any:`prepareXMLFile`
- run Marlin and catch the exit code
:return: S_OK(), S_ERROR()
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
self.log.error("Failed to resolve input parameters:", self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName)
#get the path to the detector model, either local or from the software
compactFile = None
if self.detectorModel:
resXML = self._getDetectorXML()
if not resXML['OK']:
self.log.error("Could not obtain the detector XML file: ", resXML["Message"])
return resXML
compactFile = resXML['Value']
res = getEnvironmentScript(self.platform, "marlin", self.applicationVersion, self.getEnvScript)
if not res['OK']:
self.log.error("Failed to get the env script")
return res
env_script_path = res["Value"]
res = self._getInputFiles()
if not res['OK']:
self.log.error("Failed getting input files:", res['Message'])
return res
listofslcio = res['Value']
finalXML = "marlinxml_" + self.STEP_NUMBER + ".xml"
steeringfiledirname = ''
res = getSteeringFileDirName(self.platform, "marlin", self.applicationVersion)
if res['OK']:
steeringfiledirname = res['Value']
else:
self.log.warn('Could not find the steering file directory', res['Message'])
##Handle PandoraSettings.xml
pandorasettings = 'PandoraSettings.xml'
if not os.path.exists(pandorasettings):
if steeringfiledirname and os.path.exists(os.path.join(steeringfiledirname, pandorasettings)):
try:
shutil.copy(os.path.join(steeringfiledirname, pandorasettings),
os.path.join(os.getcwd(), pandorasettings))
except EnvironmentError, x:
self.log.warn('Could not copy PandoraSettings.xml, exception: %s' % x)
if self.inputGEAR:
self.inputGEAR = os.path.basename(self.inputGEAR)
if self.inputGEAR and not os.path.exists(self.inputGEAR) and steeringfiledirname \
and os.path.exists(os.path.join(steeringfiledirname, self.inputGEAR)):
self.inputGEAR = os.path.join(steeringfiledirname, self.inputGEAR)
self.SteeringFile = os.path.basename(self.SteeringFile)
if not os.path.exists(self.SteeringFile):
if steeringfiledirname:
if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)):
self.SteeringFile = os.path.join(steeringfiledirname, self.SteeringFile)
if not self.SteeringFile:
self.log.error("Steering file not defined, shouldn't happen!")
return S_ERROR("Could not find steering file")
resOver = self._checkRunOverlay()
if not resOver['OK']:
return resOver
overlayParam = resOver['Value']
res = prepareXMLFile(finalXML, self.SteeringFile, self.inputGEAR, listofslcio,
self.NumberOfEvents, self.OutputFile, self.outputREC, self.outputDST,
self.debug,
dd4hepGeoFile=compactFile,
overlayParam=overlayParam,
)
if not res['OK']:
self.log.error('Something went wrong with XML generation because %s' % res['Message'])
self.setApplicationStatus('Marlin: something went wrong with XML generation')
return res
res = self.prepareMARLIN_DLL(env_script_path)
if not res['OK']:
self.log.error('Failed building MARLIN_DLL:', res['Message'])
self.setApplicationStatus('Failed to setup MARLIN_DLL')
return S_ERROR('Something wrong with software installation')
marlin_dll = res["Value"]
self.result = self.runMarlin(finalXML, env_script_path, marlin_dll)
if not self.result['OK']:
self.log.error('Something wrong during running:', self.result['Message'])
self.setApplicationStatus('Error during running %s' % self.applicationName)
return S_ERROR('Failed to run %s' % self.applicationName)
#self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
resultTuple = self.result['Value']
if not os.path.exists(self.applicationLog):
self.log.error("Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName))
if not self.ignoreapperrors:
return S_ERROR('%s did not produce the expected log' % (self.applicationName))
status = resultTuple[0]
# stdOutput = resultTuple[1]
# stdError = resultTuple[2]
self.log.info( "Status after the application execution is:", str( status ) )
return self.finalStatusReport(status)