def testMakeSandbox(self):
creator = SandboxCreator.SandboxCreator()
workload = TestWorkloads.twoTaskTree()
tempdir = tempfile.mkdtemp()
# test that the existing path is deleted else it will crash as in issue #5130
os.makedirs('%s/%s/WMSandbox' % (tempdir, workload.name()))
boxpath = creator.makeSandbox(tempdir, workload)
# extract our sandbox to test it
extractDir = tempfile.mkdtemp()
tarHandle = tarfile.open(boxpath, 'r:bz2')
tarHandle.extractall(extractDir)
self.fileExistsTest(extractDir + "/WMSandbox")
self.fileExistsTest(extractDir + "/WMSandbox/WMWorkload.pkl")
self.fileExistsTest(extractDir + "/WMSandbox/__init__.py")
self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/__init__.py")
self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/cmsRun1")
self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/stageOut1")
self.fileExistsTest(extractDir +
"/WMSandbox/FirstTask/cmsRun1/__init__.py")
self.fileExistsTest(extractDir +
"/WMSandbox/FirstTask/stageOut1/__init__.py")
self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/__init__.py")
self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/cmsRun2")
self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/stageOut2")
self.fileExistsTest(extractDir +
"/WMSandbox/SecondTask/cmsRun2/__init__.py")
self.fileExistsTest(extractDir +
"/WMSandbox/SecondTask/stageOut2/__init__.py")
# make sure the sandbox is there
self.fileExistsTest(extractDir + '/WMCore.zip')
# Test that zipimport works on the dummy module that SandboxCreator inserts
output = subprocess.check_output(
['python', '-m', 'WMCore.ZipImportTestModule'],
env={'PYTHONPATH': os.path.join(extractDir, 'WMCore.zip')})
self.assertIn('ZIPIMPORTTESTOK', output)
# make sure the pickled file is the same
pickleHandle = open(extractDir + "/WMSandbox/WMWorkload.pkl")
pickledWorkload = pickle.load(pickleHandle)
self.assertEqual(workload.data, pickledWorkload)
self.assertEqual(pickledWorkload.sandbox, boxpath)
#TODO:This test will be deprecated when task.data.input.sandbox property is removed
for task in workload.taskIterator():
for t in task.nodeIterator():
t = WMTask.WMTaskHelper(t)
self.assertEqual(t.data.input.sandbox, boxpath)
pickleHandle.close()
pickledWorkload.section_("test_section")
self.assertNotEqual(workload.data, pickledWorkload)
shutil.rmtree(extractDir)
shutil.rmtree(tempdir)
def testTwoTaskTree(self):
workload = TestSpecs.twoTaskTree()
tasks = workload.listAllTaskNames()
parenttask = workload.getTask("FirstTask")
findWorkload = getWorkloadFromTask(parenttask)
self.assertEqual(id(findWorkload.data), id(workload.data))
self.assertTrue( isinstance( workload, WMCore.WMSpec.WMWorkload.WMWorkloadHelper ) )
self.assertTrue( isinstance( parenttask, WMCore.WMSpec.WMTask.WMTaskHelper ) )
task = workload.getTask("SecondTask")
#print task.data
self.assertTrue( isinstance( task, WMCore.WMSpec.WMTask.WMTaskHelper ) )
steps = task.listAllStepNames()
# there should be a way to do this with iteration that would be neater
step = task.getStep(steps[0])
self.assertTrue( isinstance( step, WMCore.WMSpec.WMStep.WMStepHelper ) )
step2= task.getStep(steps[1])
self.assertTrue( isinstance( step2, WMCore.WMSpec.WMStep.WMStepHelper ) )
step3= task.getStep(steps[2])
self.assertTrue( isinstance( step3, WMCore.WMSpec.WMStep.WMStepHelper ) )
step4= task.getStep(steps[3])
self.assertTrue( isinstance( step4, WMCore.WMSpec.WMStep.WMStepHelper ) )
self.assertEqual( id(getTaskFromStep(step)), id(getTaskFromStep(step2)) )
def testTwoTaskTree(self):
workload = TestSpecs.twoTaskTree()
tasks = workload.listAllTaskNames()
parenttask = workload.getTask("FirstTask")
findWorkload = getWorkloadFromTask(parenttask)
self.assertEqual(id(findWorkload.data), id(workload.data))
self.assertTrue(
isinstance(workload, WMCore.WMSpec.WMWorkload.WMWorkloadHelper))
self.assertTrue(
isinstance(parenttask, WMCore.WMSpec.WMTask.WMTaskHelper))
task = workload.getTask("SecondTask")
#print task.data
self.assertTrue(isinstance(task, WMCore.WMSpec.WMTask.WMTaskHelper))
steps = task.listAllStepNames()
# there should be a way to do this with iteration that would be neater
step = task.getStep(steps[0])
self.assertTrue(isinstance(step, WMCore.WMSpec.WMStep.WMStepHelper))
step2 = task.getStep(steps[1])
self.assertTrue(isinstance(step2, WMCore.WMSpec.WMStep.WMStepHelper))
step3 = task.getStep(steps[2])
self.assertTrue(isinstance(step3, WMCore.WMSpec.WMStep.WMStepHelper))
step4 = task.getStep(steps[3])
self.assertTrue(isinstance(step4, WMCore.WMSpec.WMStep.WMStepHelper))
self.assertEqual(id(getTaskFromStep(step)), id(getTaskFromStep(step2)))
def testMakeSandbox(self):
creator = SandboxCreator.SandboxCreator()
workload = TestWorkloads.twoTaskTree()
tempdir = tempfile.mkdtemp()
# test that the existing path is deleted else it will crash as in issue #5130
os.makedirs("%s/%s/WMSandbox" % (tempdir, workload.name()))
boxpath = creator.makeSandbox(tempdir, workload)
# extract our sandbox to test it
extractDir = tempfile.mkdtemp()
tarHandle = tarfile.open(boxpath, "r:bz2")
tarHandle.extractall(extractDir)
self.fileExistsTest(extractDir + "/WMSandbox")
self.fileExistsTest(extractDir + "/WMSandbox/WMWorkload.pkl")
self.fileExistsTest(extractDir + "/WMSandbox/__init__.py")
self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/__init__.py")
self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/cmsRun1")
self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/stageOut1")
self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/cmsRun1/__init__.py")
self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/stageOut1/__init__.py")
self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/__init__.py")
self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/cmsRun2")
self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/stageOut2")
self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/cmsRun2/__init__.py")
self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/stageOut2/__init__.py")
# make sure the sandbox is there
self.fileExistsTest(extractDir + "/WMCore.zip")
# Test that zipimport works on the dummy module that SandboxCreator inserts
output = subprocess.check_output(
["python", "-m", "WMCore.ZipImportTestModule"], env={"PYTHONPATH": os.path.join(extractDir, "WMCore.zip")}
)
self.assertIn("ZIPIMPORTTESTOK", output)
# make sure the pickled file is the same
pickleHandle = open(extractDir + "/WMSandbox/WMWorkload.pkl")
pickledWorkload = pickle.load(pickleHandle)
self.assertEqual(workload.data, pickledWorkload)
self.assertEqual(pickledWorkload.sandbox, boxpath)
# TODO:This test will be deprecated when task.data.input.sandbox property is removed
for task in workload.taskIterator():
for t in task.nodeIterator():
t = WMTask.WMTaskHelper(t)
self.assertEqual(t.data.input.sandbox, boxpath)
pickleHandle.close()
pickledWorkload.section_("test_section")
self.assertNotEqual(workload.data, pickledWorkload)
shutil.rmtree(extractDir)
shutil.rmtree(tempdir)
def testOneTaskTwoSTep(self):
workload = TestSpecs.oneTaskTwoStep()
tasks = workload.listAllTaskNames()
task = workload.getTask(tasks[0])
findWorkload = getWorkloadFromTask(task)
self.assertEqual(id(findWorkload.data), id(workload.data))
self.assertTrue( isinstance( workload, WMCore.WMSpec.WMWorkload.WMWorkloadHelper ) )
self.assertTrue( isinstance( task, WMCore.WMSpec.WMTask.WMTaskHelper ) )
steps = task.listAllStepNames()
step = task.getStep(steps[0])
self.assertTrue( isinstance( step, WMCore.WMSpec.WMStep.WMStepHelper ) )
step2= task.getStep(steps[1])
self.assertTrue( isinstance( step2, WMCore.WMSpec.WMStep.WMStepHelper ) )
self.assertEqual( id(getTaskFromStep(step)), id(getTaskFromStep(step2)) )
def testOneTaskTwoSTep(self):
workload = TestSpecs.oneTaskTwoStep()
tasks = workload.listAllTaskNames()
task = workload.getTask(tasks[0])
findWorkload = getWorkloadFromTask(task)
self.assertEqual(id(findWorkload.data), id(workload.data))
self.assertTrue(
isinstance(workload, WMCore.WMSpec.WMWorkload.WMWorkloadHelper))
self.assertTrue(isinstance(task, WMCore.WMSpec.WMTask.WMTaskHelper))
steps = task.listAllStepNames()
step = task.getStep(steps[0])
self.assertTrue(isinstance(step, WMCore.WMSpec.WMStep.WMStepHelper))
step2 = task.getStep(steps[1])
self.assertTrue(isinstance(step2, WMCore.WMSpec.WMStep.WMStepHelper))
self.assertEqual(id(getTaskFromStep(step)), id(getTaskFromStep(step2)))
def testMakeSandbox(self):
creator = SandboxCreator.SandboxCreator()
creator.disableWMCorePackaging()
workload = TestWorkloads.twoTaskTree()
tempdir = tempfile.mkdtemp()
boxpath = creator.makeSandbox(tempdir, workload)
# extract our sandbox to test it
extractDir = tempfile.mkdtemp()
tarHandle = tarfile.open(boxpath, 'r:bz2')
tarHandle.extractall( extractDir )
self.fileExistsTest( extractDir + "/WMSandbox")
self.fileExistsTest( extractDir + "/WMSandbox/WMWorkload.pkl")
self.fileExistsTest( extractDir + "/WMSandbox/__init__.py")
self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/__init__.py")
self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/cmsRun1")
self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/stageOut1")
self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/cmsRun1/__init__.py")
self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/stageOut1/__init__.py")
self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/__init__.py")
self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/cmsRun2")
self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/stageOut2")
self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/cmsRun2/__init__.py")
self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/stageOut2/__init__.py")
# make sure the pickled file is the same
pickleHandle = open( extractDir + "/WMSandbox/WMWorkload.pkl")
pickledWorkload = pickle.load( pickleHandle )
self.assertEqual( workload.data, pickledWorkload )
self.assertEqual( pickledWorkload.sandbox, boxpath )
#TODO:This test will be deprecated when task.data.input.sandbox property is removed
for task in workload.taskIterator():
for t in task.nodeIterator():
t = WMTask.WMTaskHelper(t)
self.assertEqual(t.data.input.sandbox, boxpath)
pickleHandle.close()
pickledWorkload.section_("test_section")
self.assertNotEqual( workload.data, pickledWorkload )
shutil.rmtree( extractDir )
shutil.rmtree( tempdir )
def testMakeSandbox(self):
creator = SandboxCreator.SandboxCreator()
workload = TestWorkloads.twoTaskTree()
tempdir = tempfile.mkdtemp()
# test that the existing path is deleted else it will crash as in issue #5130
os.makedirs('%s/%s/WMSandbox' % (tempdir, workload.name()))
boxpath = creator.makeSandbox(tempdir, workload)
# extract our sandbox to test it
extractDir = tempfile.mkdtemp()
tarHandle = tarfile.open(boxpath, 'r:bz2')
tarHandle.extractall(extractDir)
self.fileExistsTest(extractDir + "/WMSandbox")
self.fileExistsTest(extractDir + "/WMSandbox/WMWorkload.pkl")
self.fileExistsTest(extractDir + "/WMSandbox/__init__.py")
self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/__init__.py")
self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/cmsRun1")
self.fileExistsTest(extractDir + "/WMSandbox/FirstTask/stageOut1")
self.fileExistsTest(extractDir +
"/WMSandbox/FirstTask/cmsRun1/__init__.py")
self.fileExistsTest(extractDir +
"/WMSandbox/FirstTask/stageOut1/__init__.py")
self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/__init__.py")
self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/cmsRun2")
self.fileExistsTest(extractDir + "/WMSandbox/SecondTask/stageOut2")
self.fileExistsTest(extractDir +
"/WMSandbox/SecondTask/cmsRun2/__init__.py")
self.fileExistsTest(extractDir +
"/WMSandbox/SecondTask/stageOut2/__init__.py")
# make sure the sandbox is there
self.fileExistsTest(extractDir + '/WMCore.zip')
# now test that zipimport works
# This gets replaced in setup/teardown
sys.path.insert(0, os.path.join(extractDir, 'WMCore.zip'))
os.system('ls -lah %s' % extractDir)
# Gotta remove this since python caches subpackage folders in package.__path__
del sys.modules['WMCore']
if 'WMCore.ZipImportTestModule' in sys.modules:
del sys.modules['WMCore.ZipImportTestModule']
import WMCore.ZipImportTestModule
sys.modules = copy.copy(self.modulesBackup)
self.assertTrue('WMCore.zip' in WMCore.ZipImportTestModule.__file__)
# make sure the pickled file is the same
pickleHandle = open(extractDir + "/WMSandbox/WMWorkload.pkl")
pickledWorkload = pickle.load(pickleHandle)
self.assertEqual(workload.data, pickledWorkload)
self.assertEqual(pickledWorkload.sandbox, boxpath)
#TODO:This test will be deprecated when task.data.input.sandbox property is removed
for task in workload.taskIterator():
for t in task.nodeIterator():
t = WMTask.WMTaskHelper(t)
self.assertEqual(t.data.input.sandbox, boxpath)
pickleHandle.close()
pickledWorkload.section_("test_section")
self.assertNotEqual(workload.data, pickledWorkload)
shutil.rmtree(extractDir)
shutil.rmtree(tempdir)
def testMakeSandbox(self):
creator = SandboxCreator.SandboxCreator()
workload = TestWorkloads.twoTaskTree()
tempdir = tempfile.mkdtemp()
boxpath = creator.makeSandbox(tempdir, workload)
# extract our sandbox to test it
extractDir = tempfile.mkdtemp()
tarHandle = tarfile.open(boxpath, 'r:bz2')
tarHandle.extractall( extractDir )
self.fileExistsTest( extractDir + "/WMSandbox")
self.fileExistsTest( extractDir + "/WMSandbox/WMWorkload.pkl")
self.fileExistsTest( extractDir + "/WMSandbox/__init__.py")
self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/__init__.py")
self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/cmsRun1")
self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/stageOut1")
self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/cmsRun1/__init__.py")
self.fileExistsTest( extractDir + "/WMSandbox/FirstTask/stageOut1/__init__.py")
self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/__init__.py")
self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/cmsRun2")
self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/stageOut2")
self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/cmsRun2/__init__.py")
self.fileExistsTest( extractDir + "/WMSandbox/SecondTask/stageOut2/__init__.py")
# make sure the sandbox is there
self.fileExistsTest( extractDir + '/WMCore.zip')
# now test that zipimport works
# This gets replaced in setup/teardown
sys.path.insert(0, os.path.join(extractDir, 'WMCore.zip'))
os.system('ls -lah %s' % extractDir)
# Gotta remove this since python caches subpackage folders in package.__path__
del sys.modules['WMCore']
if 'WMCore.ZipImportTestModule' in sys.modules:
del sys.modules['WMCore.ZipImportTestModule']
import WMCore.ZipImportTestModule
sys.modules = copy.copy(self.modulesBackup)
self.assertTrue( 'WMCore.zip' in WMCore.ZipImportTestModule.__file__ )
# make sure the pickled file is the same
pickleHandle = open( extractDir + "/WMSandbox/WMWorkload.pkl")
pickledWorkload = pickle.load( pickleHandle )
self.assertEqual( workload.data, pickledWorkload )
self.assertEqual( pickledWorkload.sandbox, boxpath )
#TODO:This test will be deprecated when task.data.input.sandbox property is removed
for task in workload.taskIterator():
for t in task.nodeIterator():
t = WMTask.WMTaskHelper(t)
self.assertEqual(t.data.input.sandbox, boxpath)
pickleHandle.close()
pickledWorkload.section_("test_section")
self.assertNotEqual( workload.data, pickledWorkload )
shutil.rmtree( extractDir )
shutil.rmtree( tempdir )
