Exemplo n.º 1
0
    def testMakeSandbox(self):
        creator = SandboxCreator.SandboxCreator()
        workload = TestWorkloads.twoTaskTree()
        tempdir = tempfile.mkdtemp()
        # test that the existing path is deleted else it will crash as in issue #5130
        os.makedirs('%s/%s/WMSandbox' % (tempdir, workload.name()))
        boxpath = creator.makeSandbox(tempdir, workload)

        # extract our sandbox to test it
        extractDir = tempfile.mkdtemp()
        tarHandle = tarfile.open(boxpath, 'r:bz2')
        tarHandle.extractall(extractDir)

        self.fileExistsTest(extractDir + "/WMSandbox")
        self.fileExistsTest(extractDir + "/WMSandbox/WMWorkload.pkl")
        self.fileExistsTest(extractDir + "/WMSandbox/__init__.py")
        self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/__init__.py")

        self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/cmsRun1")
        self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/stageOut1")
        self.fileExistsTest(extractDir +
                            "/WMSandbox/FirstTask/cmsRun1/__init__.py")
        self.fileExistsTest(extractDir +
                            "/WMSandbox/FirstTask/stageOut1/__init__.py")

        self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/__init__.py")
        self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/cmsRun2")
        self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/stageOut2")
        self.fileExistsTest(extractDir +
                            "/WMSandbox/SecondTask/cmsRun2/__init__.py")
        self.fileExistsTest(extractDir +
                            "/WMSandbox/SecondTask/stageOut2/__init__.py")

        # make sure the sandbox is there
        self.fileExistsTest(extractDir + '/WMCore.zip')

        # Test that zipimport works on the dummy module that SandboxCreator inserts
        output = subprocess.check_output(
            ['python', '-m', 'WMCore.ZipImportTestModule'],
            env={'PYTHONPATH': os.path.join(extractDir, 'WMCore.zip')})
        self.assertIn('ZIPIMPORTTESTOK', output)

        # make sure the pickled file is the same
        pickleHandle = open(extractDir + "/WMSandbox/WMWorkload.pkl")
        pickledWorkload = pickle.load(pickleHandle)
        self.assertEqual(workload.data, pickledWorkload)
        self.assertEqual(pickledWorkload.sandbox, boxpath)

        #TODO:This test will be deprecated when task.data.input.sandbox property is removed
        for task in workload.taskIterator():
            for t in task.nodeIterator():
                t = WMTask.WMTaskHelper(t)
                self.assertEqual(t.data.input.sandbox, boxpath)

        pickleHandle.close()

        pickledWorkload.section_("test_section")
        self.assertNotEqual(workload.data, pickledWorkload)
        shutil.rmtree(extractDir)
        shutil.rmtree(tempdir)
Exemplo n.º 2
0
    def testTwoTaskTree(self):
        workload = TestSpecs.twoTaskTree()
        tasks = workload.listAllTaskNames()
        parenttask = workload.getTask("FirstTask")

        findWorkload = getWorkloadFromTask(parenttask)
        self.assertEqual(id(findWorkload.data), id(workload.data))
        self.assertTrue( isinstance( workload, WMCore.WMSpec.WMWorkload.WMWorkloadHelper ) )
        self.assertTrue( isinstance( parenttask, WMCore.WMSpec.WMTask.WMTaskHelper ) )
        
        task = workload.getTask("SecondTask")
        #print task.data
        self.assertTrue( isinstance( task, WMCore.WMSpec.WMTask.WMTaskHelper ) )
        steps = task.listAllStepNames()
        
        # there should be a way to do this with iteration that would be neater
        step = task.getStep(steps[0])
        self.assertTrue( isinstance( step, WMCore.WMSpec.WMStep.WMStepHelper ) )
        step2= task.getStep(steps[1])
        self.assertTrue( isinstance( step2, WMCore.WMSpec.WMStep.WMStepHelper ) )
        step3= task.getStep(steps[2])
        self.assertTrue( isinstance( step3, WMCore.WMSpec.WMStep.WMStepHelper ) )
        step4= task.getStep(steps[3])
        self.assertTrue( isinstance( step4, WMCore.WMSpec.WMStep.WMStepHelper ) )
        self.assertEqual( id(getTaskFromStep(step)), id(getTaskFromStep(step2)) )
Exemplo n.º 3
0
    def testTwoTaskTree(self):
        workload = TestSpecs.twoTaskTree()
        tasks = workload.listAllTaskNames()
        parenttask = workload.getTask("FirstTask")

        findWorkload = getWorkloadFromTask(parenttask)
        self.assertEqual(id(findWorkload.data), id(workload.data))
        self.assertTrue(
            isinstance(workload, WMCore.WMSpec.WMWorkload.WMWorkloadHelper))
        self.assertTrue(
            isinstance(parenttask, WMCore.WMSpec.WMTask.WMTaskHelper))

        task = workload.getTask("SecondTask")
        #print task.data
        self.assertTrue(isinstance(task, WMCore.WMSpec.WMTask.WMTaskHelper))
        steps = task.listAllStepNames()

        # there should be a way to do this with iteration that would be neater
        step = task.getStep(steps[0])
        self.assertTrue(isinstance(step, WMCore.WMSpec.WMStep.WMStepHelper))
        step2 = task.getStep(steps[1])
        self.assertTrue(isinstance(step2, WMCore.WMSpec.WMStep.WMStepHelper))
        step3 = task.getStep(steps[2])
        self.assertTrue(isinstance(step3, WMCore.WMSpec.WMStep.WMStepHelper))
        step4 = task.getStep(steps[3])
        self.assertTrue(isinstance(step4, WMCore.WMSpec.WMStep.WMStepHelper))
        self.assertEqual(id(getTaskFromStep(step)), id(getTaskFromStep(step2)))
Exemplo n.º 4
0
    def testMakeSandbox(self):
        creator = SandboxCreator.SandboxCreator()
        workload = TestWorkloads.twoTaskTree()
        tempdir = tempfile.mkdtemp()
        # test that the existing path is deleted else it will crash as in issue #5130
        os.makedirs("%s/%s/WMSandbox" % (tempdir, workload.name()))
        boxpath = creator.makeSandbox(tempdir, workload)

        # extract our sandbox to test it
        extractDir = tempfile.mkdtemp()
        tarHandle = tarfile.open(boxpath, "r:bz2")
        tarHandle.extractall(extractDir)

        self.fileExistsTest(extractDir + "/WMSandbox")
        self.fileExistsTest(extractDir + "/WMSandbox/WMWorkload.pkl")
        self.fileExistsTest(extractDir + "/WMSandbox/__init__.py")
        self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/__init__.py")

        self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/cmsRun1")
        self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/stageOut1")
        self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/cmsRun1/__init__.py")
        self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/stageOut1/__init__.py")

        self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/__init__.py")
        self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/cmsRun2")
        self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/stageOut2")
        self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/cmsRun2/__init__.py")
        self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/stageOut2/__init__.py")

        # make sure the sandbox is there
        self.fileExistsTest(extractDir + "/WMCore.zip")

        # Test that zipimport works on the dummy module that SandboxCreator inserts
        output = subprocess.check_output(
            ["python", "-m", "WMCore.ZipImportTestModule"], env={"PYTHONPATH": os.path.join(extractDir, "WMCore.zip")}
        )
        self.assertIn("ZIPIMPORTTESTOK", output)

        # make sure the pickled file is the same
        pickleHandle = open(extractDir + "/WMSandbox/WMWorkload.pkl")
        pickledWorkload = pickle.load(pickleHandle)
        self.assertEqual(workload.data, pickledWorkload)
        self.assertEqual(pickledWorkload.sandbox, boxpath)

        # TODO:This test will be deprecated when task.data.input.sandbox property is removed
        for task in workload.taskIterator():
            for t in task.nodeIterator():
                t = WMTask.WMTaskHelper(t)
                self.assertEqual(t.data.input.sandbox, boxpath)

        pickleHandle.close()

        pickledWorkload.section_("test_section")
        self.assertNotEqual(workload.data, pickledWorkload)
        shutil.rmtree(extractDir)
        shutil.rmtree(tempdir)
Exemplo n.º 5
0
 def testOneTaskTwoSTep(self):
     workload = TestSpecs.oneTaskTwoStep()
     tasks = workload.listAllTaskNames()
     task = workload.getTask(tasks[0])
     findWorkload = getWorkloadFromTask(task)
     self.assertEqual(id(findWorkload.data), id(workload.data))
     self.assertTrue( isinstance( workload, WMCore.WMSpec.WMWorkload.WMWorkloadHelper ) )
     self.assertTrue( isinstance( task, WMCore.WMSpec.WMTask.WMTaskHelper ) )
     steps = task.listAllStepNames()
     step = task.getStep(steps[0])
     self.assertTrue( isinstance( step, WMCore.WMSpec.WMStep.WMStepHelper ) )
     step2= task.getStep(steps[1])
     self.assertTrue( isinstance( step2, WMCore.WMSpec.WMStep.WMStepHelper ) )
     self.assertEqual( id(getTaskFromStep(step)), id(getTaskFromStep(step2)) )
Exemplo n.º 6
0
 def testOneTaskTwoSTep(self):
     workload = TestSpecs.oneTaskTwoStep()
     tasks = workload.listAllTaskNames()
     task = workload.getTask(tasks[0])
     findWorkload = getWorkloadFromTask(task)
     self.assertEqual(id(findWorkload.data), id(workload.data))
     self.assertTrue(
         isinstance(workload, WMCore.WMSpec.WMWorkload.WMWorkloadHelper))
     self.assertTrue(isinstance(task, WMCore.WMSpec.WMTask.WMTaskHelper))
     steps = task.listAllStepNames()
     step = task.getStep(steps[0])
     self.assertTrue(isinstance(step, WMCore.WMSpec.WMStep.WMStepHelper))
     step2 = task.getStep(steps[1])
     self.assertTrue(isinstance(step2, WMCore.WMSpec.WMStep.WMStepHelper))
     self.assertEqual(id(getTaskFromStep(step)), id(getTaskFromStep(step2)))
Exemplo n.º 7
0
 def testMakeSandbox(self):
     creator  = SandboxCreator.SandboxCreator()
     creator.disableWMCorePackaging()
     workload = TestWorkloads.twoTaskTree()
     tempdir  = tempfile.mkdtemp()
     boxpath  = creator.makeSandbox(tempdir, workload)
     
     # extract our sandbox to test it
     extractDir = tempfile.mkdtemp()
     tarHandle  = tarfile.open(boxpath, 'r:bz2')
     tarHandle.extractall( extractDir )
     
     self.fileExistsTest( extractDir + "/WMSandbox")
     self.fileExistsTest( extractDir + "/WMSandbox/WMWorkload.pkl")
     self.fileExistsTest( extractDir + "/WMSandbox/__init__.py")
     self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/__init__.py")
     
     self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/cmsRun1")
     self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/stageOut1")
     self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/cmsRun1/__init__.py")
     self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/stageOut1/__init__.py")
     
     self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/__init__.py")
     self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/cmsRun2")
     self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/stageOut2")
     self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/cmsRun2/__init__.py")
     self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/stageOut2/__init__.py")
     
     # make sure the pickled file is the same
     pickleHandle = open( extractDir + "/WMSandbox/WMWorkload.pkl")
     pickledWorkload = pickle.load( pickleHandle )
     self.assertEqual( workload.data, pickledWorkload )
     self.assertEqual( pickledWorkload.sandbox, boxpath )
     
     #TODO:This test will be deprecated when task.data.input.sandbox property is removed
     for task in workload.taskIterator():
         for t in task.nodeIterator():
             t = WMTask.WMTaskHelper(t)
             self.assertEqual(t.data.input.sandbox, boxpath)
             
     pickleHandle.close()
     
     pickledWorkload.section_("test_section")
     self.assertNotEqual( workload.data, pickledWorkload )
     shutil.rmtree( extractDir )
     shutil.rmtree( tempdir )
Exemplo n.º 8
0
    def testMakeSandbox(self):
        creator = SandboxCreator.SandboxCreator()
        workload = TestWorkloads.twoTaskTree()
        tempdir = tempfile.mkdtemp()
        # test that the existing path is deleted else it will crash as in issue #5130
        os.makedirs('%s/%s/WMSandbox' % (tempdir, workload.name()))
        boxpath = creator.makeSandbox(tempdir, workload)

        # extract our sandbox to test it
        extractDir = tempfile.mkdtemp()
        tarHandle = tarfile.open(boxpath, 'r:bz2')
        tarHandle.extractall(extractDir)

        self.fileExistsTest(extractDir + "/WMSandbox")
        self.fileExistsTest(extractDir + "/WMSandbox/WMWorkload.pkl")
        self.fileExistsTest(extractDir + "/WMSandbox/__init__.py")
        self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/__init__.py")

        self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/cmsRun1")
        self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/stageOut1")
        self.fileExistsTest(extractDir +
                            "/WMSandbox/FirstTask/cmsRun1/__init__.py")
        self.fileExistsTest(extractDir +
                            "/WMSandbox/FirstTask/stageOut1/__init__.py")

        self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/__init__.py")
        self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/cmsRun2")
        self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/stageOut2")
        self.fileExistsTest(extractDir +
                            "/WMSandbox/SecondTask/cmsRun2/__init__.py")
        self.fileExistsTest(extractDir +
                            "/WMSandbox/SecondTask/stageOut2/__init__.py")

        # make sure the sandbox is there
        self.fileExistsTest(extractDir + '/WMCore.zip')

        # now test that zipimport works
        # This gets replaced in setup/teardown
        sys.path.insert(0, os.path.join(extractDir, 'WMCore.zip'))
        os.system('ls -lah %s' % extractDir)
        # Gotta remove this since python caches subpackage folders in package.__path__
        del sys.modules['WMCore']
        if 'WMCore.ZipImportTestModule' in sys.modules:
            del sys.modules['WMCore.ZipImportTestModule']

        import WMCore.ZipImportTestModule
        sys.modules = copy.copy(self.modulesBackup)
        self.assertTrue('WMCore.zip' in WMCore.ZipImportTestModule.__file__)

        # make sure the pickled file is the same
        pickleHandle = open(extractDir + "/WMSandbox/WMWorkload.pkl")
        pickledWorkload = pickle.load(pickleHandle)
        self.assertEqual(workload.data, pickledWorkload)
        self.assertEqual(pickledWorkload.sandbox, boxpath)

        #TODO:This test will be deprecated when task.data.input.sandbox property is removed
        for task in workload.taskIterator():
            for t in task.nodeIterator():
                t = WMTask.WMTaskHelper(t)
                self.assertEqual(t.data.input.sandbox, boxpath)

        pickleHandle.close()

        pickledWorkload.section_("test_section")
        self.assertNotEqual(workload.data, pickledWorkload)
        shutil.rmtree(extractDir)
        shutil.rmtree(tempdir)
Exemplo n.º 9
0
    def testMakeSandbox(self):
        creator  = SandboxCreator.SandboxCreator()
        workload = TestWorkloads.twoTaskTree()
        tempdir  = tempfile.mkdtemp()
        boxpath  = creator.makeSandbox(tempdir, workload)
        
        # extract our sandbox to test it
        extractDir = tempfile.mkdtemp()
        tarHandle  = tarfile.open(boxpath, 'r:bz2')
        tarHandle.extractall( extractDir )
        
        self.fileExistsTest( extractDir + "/WMSandbox")
        self.fileExistsTest( extractDir + "/WMSandbox/WMWorkload.pkl")
        self.fileExistsTest( extractDir + "/WMSandbox/__init__.py")
        self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/__init__.py")
        
        self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/cmsRun1")
        self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/stageOut1")
        self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/cmsRun1/__init__.py")
        self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/stageOut1/__init__.py")
        
        self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/__init__.py")
        self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/cmsRun2")
        self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/stageOut2")
        self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/cmsRun2/__init__.py")
        self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/stageOut2/__init__.py")
        
        # make sure the sandbox is there
        self.fileExistsTest( extractDir + '/WMCore.zip')

        # now test that zipimport works
        # This gets replaced in setup/teardown
        sys.path.insert(0, os.path.join(extractDir, 'WMCore.zip'))
        os.system('ls -lah %s' % extractDir)
        # Gotta remove this since python caches subpackage folders in package.__path__
        del sys.modules['WMCore']
        if 'WMCore.ZipImportTestModule' in sys.modules:
            del sys.modules['WMCore.ZipImportTestModule']
            
        import WMCore.ZipImportTestModule
        sys.modules = copy.copy(self.modulesBackup)
        self.assertTrue( 'WMCore.zip' in WMCore.ZipImportTestModule.__file__ )
        
        # make sure the pickled file is the same
        pickleHandle = open( extractDir + "/WMSandbox/WMWorkload.pkl")
        pickledWorkload = pickle.load( pickleHandle )
        self.assertEqual( workload.data, pickledWorkload )
        self.assertEqual( pickledWorkload.sandbox, boxpath )
        
        #TODO:This test will be deprecated when task.data.input.sandbox property is removed
        for task in workload.taskIterator():
            for t in task.nodeIterator():
                t = WMTask.WMTaskHelper(t)
                self.assertEqual(t.data.input.sandbox, boxpath)
                
        pickleHandle.close()
        
        pickledWorkload.section_("test_section")
        self.assertNotEqual( workload.data, pickledWorkload )
        shutil.rmtree( extractDir )
        shutil.rmtree( tempdir )