コード例 #1
0
ファイル: CMSSW_t.py プロジェクト: sharmaprajesh/WMCore
    def testA_Execute(self):
        """
        _Execute_

        Test the full path, including execute.
        """
        try:
            os.chdir(self.step.builder.workingDir)
            executor = CMSSWExecutor()
            executor.initialise(self.step, self.job)
            executor.pre()
            # Remove the scram pre-script as it requires an actual SCRAM environment
            executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset")
            executor.execute()
            executor.post()
            # Check that there were no errors
            self.assertEqual(0, executor.report.getExitCode())
            # Check that we processed the XML
            self.assertEqual(2, len(executor.report.getAllFiles()))
            # Check that the executable was really executed
            self.assertTrue(
                os.path.isfile(
                    os.path.join(self.step.builder.workingDir,
                                 "BogusFile.txt")))
        except Exception as ex:
            self.fail("Failure encountered, %s" % str(ex))
        finally:
            os.chdir(self.oldCwd)
        return
コード例 #2
0
ファイル: CMSSW_t.py プロジェクト: vkuznet/WMCore
    def testA_Execute(self):
        """
        _Execute_

        Test the full path, including execute.
        """
        try:
            os.chdir(self.step.builder.workingDir)
            executor = CMSSWExecutor()
            executor.initialise(self.step, self.job)
            executor.pre()
            # Remove the scram pre-script as it requires an actual SCRAM environment
            executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset")
            executor.execute()
            executor.post()
            # Check that there were no errors
            self.assertEqual(0, executor.report.getExitCode())
            # Check that we processed the XML
            self.assertEqual(2, len(executor.report.getAllFiles()))
            # Check that the executable was really executed
            self.assertTrue(os.path.isfile(os.path.join(self.step.builder.workingDir, "BogusFile.txt")))
        except Exception as ex:
            self.fail("Failure encountered, %s" % str(ex))
        finally:
            os.chdir(self.oldCwd)
        return
コード例 #3
0
ファイル: CMSSW_t.py プロジェクト: ticoann/WMCore
    def testB_Execute(self):
        """
        _Execute_

        This test will only run where the scram setup can be done properly.

        To be honest it might have to be updated, but I don't know how.
        For now I'm skipping it.
        """

        executor = CMSSWExecutor()
        executor.initialise(self.step, self.job)
        executor.pre()
        try:
            executor.execute()
        except WMExecutionFailure:
            # This fails for me now
            # If it does so, just return
            return
        executor.post()

        print executor.report
        return
コード例 #4
0
    def testB_Execute(self):
        """
        _Execute_

        This test will only run where the scram setup can be done properly.

        To be honest it might have to be updated, but I don't know how.
        For now I'm skipping it.
        """

        executor = CMSSWExecutor()
        executor.initialise(self.step, self.job)
        executor.pre()
        try:
            executor.execute()
        except WMExecutionFailure:
            # This fails for me now
            # If it does so, just return
            return
        executor.post()

        print executor.report
        return
コード例 #5
0
ファイル: CMSSW_t.py プロジェクト: ticoann/WMCore
    def testA_PrePost(self):
        """
        _PrePost_

        This test should run on any machine.
        It will simply make sure there are no major syntax errors or
        incompatibilities.

        It does not test the main functionality.
        """
        executor = CMSSWExecutor()
        executor.initialise(self.step, self.job)
        executor.pre()
        #executor.execute()
        executor.post()
        self.assertEqual(executor.report.data.ExecutorTest.status, 1)
        self.assertEqual(executor.report.data.ExecutorTest.analysis.files.fileCount, 0)
        return
コード例 #6
0
    def testA_PrePost(self):
        """
        _PrePost_

        This test should run on any machine.
        It will simply make sure there are no major syntax errors or
        incompatibilities.

        It does not test the main functionality.
        """
        executor = CMSSWExecutor()
        executor.initialise(self.step, self.job)
        executor.pre()
        #executor.execute()
        executor.post()
        self.assertEqual(executor.report.data.ExecutorTest.status, 1)
        self.assertEqual(
            executor.report.data.ExecutorTest.analysis.files.fileCount, 0)
        return
コード例 #7
0
    def testScramArchParsing(self):
        """
        Test the various modes f parsing for the scram arch
        """

        try:
            os.chdir(self.step.builder.workingDir)
            executor = CMSSWExecutor()
            with tempfile.NamedTemporaryFile() as tf:
                tf.write('GLIDEIN_REQUIRED_OS = "rhel6" \n')
                tf.write('Memory = 2048\n')
                tf.flush()
                with tmpEnv(_CONDOR_MACHINE_AD=tf.name):
                    self.assertEquals(executor.getSingleScramArch('slc6_blah_blah'), 'slc6_blah_blah')
                    self.assertEquals(executor.getSingleScramArch('slc5_blah_blah'), 'slc5_blah_blah')
                    self.assertEquals(executor.getSingleScramArch(['slc6_blah_blah', 'slc7_blah_blah']),
                                      'slc6_blah_blah')
                    self.assertEquals(executor.getSingleScramArch(['slc6_blah_blah', 'slc5_blah_blah']),
                                      'slc6_blah_blah')
                    self.assertEquals(executor.getSingleScramArch(['slc7_blah_blah', 'slc8_blah_blah']), None)
            with tempfile.NamedTemporaryFile() as tf:
                tf.write('GLIDEIN_REQUIRED_OS = "rhel7" \n')
                tf.write('Memory = 2048\n')
                tf.flush()
                with tmpEnv(_CONDOR_MACHINE_AD=tf.name):
                    self.assertEquals(executor.getSingleScramArch('slc6_blah_blah'), 'slc6_blah_blah')
                    self.assertEquals(executor.getSingleScramArch('slc7_blah_blah'), 'slc7_blah_blah')
                    self.assertEquals(executor.getSingleScramArch(['slc6_blah_blah', 'slc7_blah_blah']),
                                      'slc7_blah_blah')
                    self.assertEquals(executor.getSingleScramArch(['slc6_blah_blah', 'slc5_blah_blah']), None)
                    self.assertEquals(executor.getSingleScramArch(['slc7_blah_blah', 'slc8_blah_blah']),
                                      'slc7_blah_blah')
        except Exception:
            raise
        finally:
            os.chdir(self.oldCwd)
        return
コード例 #8
0
def executeCMSSWStack(opts, scram):

    def getOutputModules():
        pythonScript = "from PSetTweaks.WMTweak import makeTweak;"+\
                       "config = __import__(\"WMTaskSpace.cmsRun.PSet\", globals(), locals(), [\"process\"], -1);"+\
                       "tweakJson = makeTweak(config.process).jsondictionary();"+\
                       "print tweakJson[\"process\"][\"outputModules_\"]"
        ret = scram("python -c '%s'" % pythonScript, logName=subprocess.PIPE, runtimeDir=os.getcwd())
        if ret > 0:
            msg = scram.diagnostic()
            handleException("FAILED", EC_CMSRunWrapper, 'Error getting output modules from the pset.\n\tScram Env %s\n\tCommand:%s' % (msg, pythonScript))
            mintime()
            sys.exit(EC_CMSRunWrapper)
        return literal_eval(scram.stdout)

    cmssw = CMSSW()
    cmssw.stepName = "cmsRun"
    cmssw.step = WMStep(cmssw.stepName)
    CMSSWTemplate().install(cmssw.step)
    cmssw.task = makeWMTask(cmssw.stepName)
    cmssw.workload = newWorkload(cmssw.stepName)
    cmssw.step.application.setup.softwareEnvironment = ''
    cmssw.step.application.setup.scramArch = opts.scramArch
    cmssw.step.application.setup.cmsswVersion = opts.cmsswVersion
    cmssw.step.application.configuration.section_("arguments")
    cmssw.step.application.configuration.arguments.globalTag = ""
    for output in getOutputModules():
        cmssw.step.output.modules.section_(output)
        getattr(cmssw.step.output.modules, output).primaryDataset   = ''
        getattr(cmssw.step.output.modules, output).processedDataset = ''
        getattr(cmssw.step.output.modules, output).dataTier         = ''
    #cmssw.step.application.command.arguments = '' #TODO
    cmssw.step.user.inputSandboxes = [opts.archiveJob]
    cmssw.step.user.userFiles = opts.userFiles or ''
    #Setting the following job attribute is required because in the CMSSW executor there is a call to analysisFileLFN to set up some attributes for TFiles.
    #Same for lfnbase. We actually don't use these information so I am setting these to dummy values. Next: fix and use this lfn or drop WMCore runtime..
    cmssw.job = {'counter' : 0, 'workflow' : 'unused'}
    cmssw.step.user.lfnBase = '/store/temp/user/'
    cmssw.step.section_("builder")
    cmssw.step.builder.workingDir = os.getcwd()
    cmssw.step.runtime.invokeCommand = 'python'
    cmssw.step.runtime.scramPreDir = os.getcwd()
    cmssw.step.runtime.preScripts = []


    cmssw.step.runtime.scramPreScripts = [('%s/TweakPSet.py --location=%s '+
                                                          '--inputFile=\'%s\' '+
                                                          '--runAndLumis=\'%s\' '+
                                                          '--firstEvent=%s '+
                                                          '--lastEvent=%s '+
                                                          '--firstLumi=%s '+
                                                          '--firstRun=%s '+
                                                          '--seeding=%s '+
                                                          '--lheInputFiles=%s '+
                                                          '--oneEventMode=%s ' +
                                                          '--eventsPerLumi=%s') %
                                             (os.getcwd(), os.getcwd(),
                                                           opts.inputFile,
                                                           opts.runAndLumis,
                                                           opts.firstEvent,
                                                           opts.lastEvent,
                                                           opts.firstLumi,
                                                           opts.firstRun,
                                                           opts.seeding,
                                                           opts.lheInputFiles,
                                                           opts.oneEventMode,
                                                           opts.eventsPerLumi)]
    cmssw.step.section_("execution") #exitStatus of cmsRun is set here
    cmssw.report = Report("cmsRun") #report is loaded and put here
    cmssw.execute()
    return cmssw
コード例 #9
0
def executeCMSSWStack(opts, scram):
    def getOutputModules():
        pythonScript = "from PSetTweaks.WMTweak import makeTweak;"+\
                       "config = __import__(\"WMTaskSpace.cmsRun.PSet\", globals(), locals(), [\"process\"], -1);"+\
                       "tweakJson = makeTweak(config.process).jsondictionary();"+\
                       "print tweakJson[\"process\"][\"outputModules_\"]"
        with tempSetLogLevel(logger=logging.getLogger(), level=logging.ERROR):
            ret = scram("python -c '%s'" % pythonScript,
                        runtimeDir=os.getcwd())
        if ret > 0:
            msg = 'Error getting output modules from the pset.\n\tScram Diagnostic %s' % scram.diagnostic(
            )
            handleException("FAILED", EC_CMSRunWrapper, msg)
            mintime()
            sys.exit(EC_CMSRunWrapper)
        output = literal_eval(scram.getStdout())
        return output

    cmssw = CMSSW()
    cmssw.stepName = "cmsRun"
    cmssw.step = WMStep(cmssw.stepName)
    CMSSWTemplate().install(cmssw.step)
    cmssw.task = makeWMTask(cmssw.stepName)
    cmssw.workload = newWorkload(cmssw.stepName)
    cmssw.step.application.setup.softwareEnvironment = ''
    cmssw.step.application.setup.scramArch = opts.scramArch
    cmssw.step.application.setup.cmsswVersion = opts.cmsswVersion
    cmssw.step.application.configuration.section_("arguments")
    cmssw.step.application.configuration.arguments.globalTag = ""
    for output in getOutputModules():
        cmssw.step.output.modules.section_(output)
        getattr(cmssw.step.output.modules, output).primaryDataset = ''
        getattr(cmssw.step.output.modules, output).processedDataset = ''
        getattr(cmssw.step.output.modules, output).dataTier = ''
    #cmssw.step.application.command.arguments = '' #TODO
    cmssw.step.user.inputSandboxes = [opts.archiveJob]
    cmssw.step.user.userFiles = opts.userFiles or ''
    #Setting the following job attribute is required because in the CMSSW executor there is a call to analysisFileLFN to set up some attributes for TFiles.
    #Same for lfnbase. We actually don't use these information so I am setting these to dummy values. Next: fix and use this lfn or drop WMCore runtime..
    cmssw.job = {'counter': 0, 'workflow': 'unused'}
    cmssw.step.user.lfnBase = '/store/temp/user/'
    cmssw.step.section_("builder")
    cmssw.step.builder.workingDir = os.getcwd()
    cmssw.step.runtime.invokeCommand = 'python'
    cmssw.step.runtime.scramPreDir = os.getcwd()
    cmssw.step.runtime.preScripts = []

    cmssw.step.runtime.scramPreScripts = [
        ('%s/TweakPSet.py --location=%s ' + '--inputFile=\'%s\' ' +
         '--runAndLumis=\'%s\' ' + '--firstEvent=%s ' + '--lastEvent=%s ' +
         '--firstLumi=%s ' + '--firstRun=%s ' + '--seeding=%s ' +
         '--lheInputFiles=%s ' + '--oneEventMode=%s ' + '--eventsPerLumi=%s ' +
         '--maxRuntime=%s') %
        (os.getcwd(), os.getcwd(), opts.inputFile, opts.runAndLumis,
         opts.firstEvent, opts.lastEvent, opts.firstLumi, opts.firstRun,
         opts.seeding, opts.lheInputFiles, opts.oneEventMode,
         opts.eventsPerLumi, opts.maxRuntime)
    ]
    cmssw.step.section_("execution")  #exitStatus of cmsRun is set here
    cmssw.report = Report("cmsRun")  #report is loaded and put here
    cmssw.execute()
    return cmssw