def testMakeSandbox(self): creator = SandboxCreator.SandboxCreator() workload = TestWorkloads.twoTaskTree() tempdir = tempfile.mkdtemp() # test that the existing path is deleted else it will crash as in issue #5130 os.makedirs('%s/%s/WMSandbox' % (tempdir, workload.name())) boxpath = creator.makeSandbox(tempdir, workload) # extract our sandbox to test it extractDir = tempfile.mkdtemp() tarHandle = tarfile.open(boxpath, 'r:bz2') tarHandle.extractall(extractDir) self.fileExistsTest(extractDir + "/WMSandbox") self.fileExistsTest(extractDir + "/WMSandbox/WMWorkload.pkl") self.fileExistsTest(extractDir + "/WMSandbox/__init__.py") self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/__init__.py") self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/cmsRun1") self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/stageOut1") self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/cmsRun1/__init__.py") self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/stageOut1/__init__.py") self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/__init__.py") self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/cmsRun2") self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/stageOut2") self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/cmsRun2/__init__.py") self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/stageOut2/__init__.py") # make sure the sandbox is there self.fileExistsTest(extractDir + '/WMCore.zip') # Test that zipimport works on the dummy module that SandboxCreator inserts output = subprocess.check_output( ['python', '-m', 'WMCore.ZipImportTestModule'], env={'PYTHONPATH': os.path.join(extractDir, 'WMCore.zip')}) self.assertIn('ZIPIMPORTTESTOK', output) # make sure the pickled file is the same pickleHandle = open(extractDir + "/WMSandbox/WMWorkload.pkl") pickledWorkload = pickle.load(pickleHandle) self.assertEqual(workload.data, pickledWorkload) self.assertEqual(pickledWorkload.sandbox, boxpath) #TODO:This test will be deprecated when task.data.input.sandbox property is removed for task in workload.taskIterator(): for t in task.nodeIterator(): t = WMTask.WMTaskHelper(t) self.assertEqual(t.data.input.sandbox, boxpath) pickleHandle.close() pickledWorkload.section_("test_section") self.assertNotEqual(workload.data, pickledWorkload) shutil.rmtree(extractDir) shutil.rmtree(tempdir)
def testTwoTaskTree(self): workload = TestSpecs.twoTaskTree() tasks = workload.listAllTaskNames() parenttask = workload.getTask("FirstTask") findWorkload = getWorkloadFromTask(parenttask) self.assertEqual(id(findWorkload.data), id(workload.data)) self.assertTrue( isinstance( workload, WMCore.WMSpec.WMWorkload.WMWorkloadHelper ) ) self.assertTrue( isinstance( parenttask, WMCore.WMSpec.WMTask.WMTaskHelper ) ) task = workload.getTask("SecondTask") #print task.data self.assertTrue( isinstance( task, WMCore.WMSpec.WMTask.WMTaskHelper ) ) steps = task.listAllStepNames() # there should be a way to do this with iteration that would be neater step = task.getStep(steps[0]) self.assertTrue( isinstance( step, WMCore.WMSpec.WMStep.WMStepHelper ) ) step2= task.getStep(steps[1]) self.assertTrue( isinstance( step2, WMCore.WMSpec.WMStep.WMStepHelper ) ) step3= task.getStep(steps[2]) self.assertTrue( isinstance( step3, WMCore.WMSpec.WMStep.WMStepHelper ) ) step4= task.getStep(steps[3]) self.assertTrue( isinstance( step4, WMCore.WMSpec.WMStep.WMStepHelper ) ) self.assertEqual( id(getTaskFromStep(step)), id(getTaskFromStep(step2)) )
def testTwoTaskTree(self): workload = TestSpecs.twoTaskTree() tasks = workload.listAllTaskNames() parenttask = workload.getTask("FirstTask") findWorkload = getWorkloadFromTask(parenttask) self.assertEqual(id(findWorkload.data), id(workload.data)) self.assertTrue( isinstance(workload, WMCore.WMSpec.WMWorkload.WMWorkloadHelper)) self.assertTrue( isinstance(parenttask, WMCore.WMSpec.WMTask.WMTaskHelper)) task = workload.getTask("SecondTask") #print task.data self.assertTrue(isinstance(task, WMCore.WMSpec.WMTask.WMTaskHelper)) steps = task.listAllStepNames() # there should be a way to do this with iteration that would be neater step = task.getStep(steps[0]) self.assertTrue(isinstance(step, WMCore.WMSpec.WMStep.WMStepHelper)) step2 = task.getStep(steps[1]) self.assertTrue(isinstance(step2, WMCore.WMSpec.WMStep.WMStepHelper)) step3 = task.getStep(steps[2]) self.assertTrue(isinstance(step3, WMCore.WMSpec.WMStep.WMStepHelper)) step4 = task.getStep(steps[3]) self.assertTrue(isinstance(step4, WMCore.WMSpec.WMStep.WMStepHelper)) self.assertEqual(id(getTaskFromStep(step)), id(getTaskFromStep(step2)))
def testMakeSandbox(self): creator = SandboxCreator.SandboxCreator() workload = TestWorkloads.twoTaskTree() tempdir = tempfile.mkdtemp() # test that the existing path is deleted else it will crash as in issue #5130 os.makedirs("%s/%s/WMSandbox" % (tempdir, workload.name())) boxpath = creator.makeSandbox(tempdir, workload) # extract our sandbox to test it extractDir = tempfile.mkdtemp() tarHandle = tarfile.open(boxpath, "r:bz2") tarHandle.extractall(extractDir) self.fileExistsTest(extractDir + "/WMSandbox") self.fileExistsTest(extractDir + "/WMSandbox/WMWorkload.pkl") self.fileExistsTest(extractDir + "/WMSandbox/__init__.py") self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/__init__.py") self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/cmsRun1") self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/stageOut1") self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/cmsRun1/__init__.py") self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/stageOut1/__init__.py") self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/__init__.py") self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/cmsRun2") self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/stageOut2") self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/cmsRun2/__init__.py") self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/stageOut2/__init__.py") # make sure the sandbox is there self.fileExistsTest(extractDir + "/WMCore.zip") # Test that zipimport works on the dummy module that SandboxCreator inserts output = subprocess.check_output( ["python", "-m", "WMCore.ZipImportTestModule"], env={"PYTHONPATH": os.path.join(extractDir, "WMCore.zip")} ) self.assertIn("ZIPIMPORTTESTOK", output) # make sure the pickled file is the same pickleHandle = open(extractDir + "/WMSandbox/WMWorkload.pkl") pickledWorkload = pickle.load(pickleHandle) self.assertEqual(workload.data, pickledWorkload) self.assertEqual(pickledWorkload.sandbox, boxpath) # TODO:This test will be deprecated when task.data.input.sandbox property is removed for task in workload.taskIterator(): for t in task.nodeIterator(): t = WMTask.WMTaskHelper(t) self.assertEqual(t.data.input.sandbox, boxpath) pickleHandle.close() pickledWorkload.section_("test_section") self.assertNotEqual(workload.data, pickledWorkload) shutil.rmtree(extractDir) shutil.rmtree(tempdir)
def testMakeSandbox(self): creator = SandboxCreator.SandboxCreator() creator.disableWMCorePackaging() workload = TestWorkloads.twoTaskTree() tempdir = tempfile.mkdtemp() boxpath = creator.makeSandbox(tempdir, workload) # extract our sandbox to test it extractDir = tempfile.mkdtemp() tarHandle = tarfile.open(boxpath, 'r:bz2') tarHandle.extractall( extractDir ) self.fileExistsTest( extractDir + "/WMSandbox") self.fileExistsTest( extractDir + "/WMSandbox/WMWorkload.pkl") self.fileExistsTest( extractDir + "/WMSandbox/__init__.py") self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/__init__.py") self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/cmsRun1") self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/stageOut1") self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/cmsRun1/__init__.py") self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/stageOut1/__init__.py") self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/__init__.py") self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/cmsRun2") self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/stageOut2") self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/cmsRun2/__init__.py") self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/stageOut2/__init__.py") # make sure the pickled file is the same pickleHandle = open( extractDir + "/WMSandbox/WMWorkload.pkl") pickledWorkload = pickle.load( pickleHandle ) self.assertEqual( workload.data, pickledWorkload ) self.assertEqual( pickledWorkload.sandbox, boxpath ) #TODO:This test will be deprecated when task.data.input.sandbox property is removed for task in workload.taskIterator(): for t in task.nodeIterator(): t = WMTask.WMTaskHelper(t) self.assertEqual(t.data.input.sandbox, boxpath) pickleHandle.close() pickledWorkload.section_("test_section") self.assertNotEqual( workload.data, pickledWorkload ) shutil.rmtree( extractDir ) shutil.rmtree( tempdir )
def testMakeSandbox(self): creator = SandboxCreator.SandboxCreator() workload = TestWorkloads.twoTaskTree() tempdir = tempfile.mkdtemp() # test that the existing path is deleted else it will crash as in issue #5130 os.makedirs('%s/%s/WMSandbox' % (tempdir, workload.name())) boxpath = creator.makeSandbox(tempdir, workload) # extract our sandbox to test it extractDir = tempfile.mkdtemp() tarHandle = tarfile.open(boxpath, 'r:bz2') tarHandle.extractall(extractDir) self.fileExistsTest(extractDir + "/WMSandbox") self.fileExistsTest(extractDir + "/WMSandbox/WMWorkload.pkl") self.fileExistsTest(extractDir + "/WMSandbox/__init__.py") self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/__init__.py") self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/cmsRun1") self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/stageOut1") self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/cmsRun1/__init__.py") self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/stageOut1/__init__.py") self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/__init__.py") self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/cmsRun2") self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/stageOut2") self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/cmsRun2/__init__.py") self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/stageOut2/__init__.py") # make sure the sandbox is there self.fileExistsTest(extractDir + '/WMCore.zip') # now test that zipimport works # This gets replaced in setup/teardown sys.path.insert(0, os.path.join(extractDir, 'WMCore.zip')) os.system('ls -lah %s' % extractDir) # Gotta remove this since python caches subpackage folders in package.__path__ del sys.modules['WMCore'] if 'WMCore.ZipImportTestModule' in sys.modules: del sys.modules['WMCore.ZipImportTestModule'] import WMCore.ZipImportTestModule sys.modules = copy.copy(self.modulesBackup) self.assertTrue('WMCore.zip' in WMCore.ZipImportTestModule.__file__) # make sure the pickled file is the same pickleHandle = open(extractDir + "/WMSandbox/WMWorkload.pkl") pickledWorkload = pickle.load(pickleHandle) self.assertEqual(workload.data, pickledWorkload) self.assertEqual(pickledWorkload.sandbox, boxpath) #TODO:This test will be deprecated when task.data.input.sandbox property is removed for task in workload.taskIterator(): for t in task.nodeIterator(): t = WMTask.WMTaskHelper(t) self.assertEqual(t.data.input.sandbox, boxpath) pickleHandle.close() pickledWorkload.section_("test_section") self.assertNotEqual(workload.data, pickledWorkload) shutil.rmtree(extractDir) shutil.rmtree(tempdir)
def testMakeSandbox(self): creator = SandboxCreator.SandboxCreator() workload = TestWorkloads.twoTaskTree() tempdir = tempfile.mkdtemp() boxpath = creator.makeSandbox(tempdir, workload) # extract our sandbox to test it extractDir = tempfile.mkdtemp() tarHandle = tarfile.open(boxpath, 'r:bz2') tarHandle.extractall( extractDir ) self.fileExistsTest( extractDir + "/WMSandbox") self.fileExistsTest( extractDir + "/WMSandbox/WMWorkload.pkl") self.fileExistsTest( extractDir + "/WMSandbox/__init__.py") self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/__init__.py") self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/cmsRun1") self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/stageOut1") self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/cmsRun1/__init__.py") self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/stageOut1/__init__.py") self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/__init__.py") self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/cmsRun2") self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/stageOut2") self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/cmsRun2/__init__.py") self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/stageOut2/__init__.py") # make sure the sandbox is there self.fileExistsTest( extractDir + '/WMCore.zip') # now test that zipimport works # This gets replaced in setup/teardown sys.path.insert(0, os.path.join(extractDir, 'WMCore.zip')) os.system('ls -lah %s' % extractDir) # Gotta remove this since python caches subpackage folders in package.__path__ del sys.modules['WMCore'] if 'WMCore.ZipImportTestModule' in sys.modules: del sys.modules['WMCore.ZipImportTestModule'] import WMCore.ZipImportTestModule sys.modules = copy.copy(self.modulesBackup) self.assertTrue( 'WMCore.zip' in WMCore.ZipImportTestModule.__file__ ) # make sure the pickled file is the same pickleHandle = open( extractDir + "/WMSandbox/WMWorkload.pkl") pickledWorkload = pickle.load( pickleHandle ) self.assertEqual( workload.data, pickledWorkload ) self.assertEqual( pickledWorkload.sandbox, boxpath ) #TODO:This test will be deprecated when task.data.input.sandbox property is removed for task in workload.taskIterator(): for t in task.nodeIterator(): t = WMTask.WMTaskHelper(t) self.assertEqual(t.data.input.sandbox, boxpath) pickleHandle.close() pickledWorkload.section_("test_section") self.assertNotEqual( workload.data, pickledWorkload ) shutil.rmtree( extractDir ) shutil.rmtree( tempdir )