class TestCase(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.testNo = TestStatus.getTestSetup(1, 1, 5, 5)
self.depth = TestStatus.getTestSetup(1, 2, 2, 3)
self.jobTreeDir = os.getcwd() + "/jobTree" #A directory for the job tree to be created in
self.tempFileTreeDir = os.path.join(os.getcwd(), "tempFileTree")
self.tempFileTree = TempFileTree(self.tempFileTreeDir) #A place to get temp files from
parasolRestart()
def tearDown(self):
unittest.TestCase.tearDown(self)
self.tempFileTree.destroyTempFiles()
parasolStop()
parasolRestart()
system("rm -rf %s %s" % (self.jobTreeDir, self.tempFileTreeDir)) #Cleanup the job tree in case it hasn't already been cleaned up.
def testJobTree_Parasol(self):
"""Runs a test program using the job tree, whilst constantly restarting parasol
by killing the nodes.
"""
for test in xrange(self.testNo): #Does not run this test when doing short testing
jobTreeCommand, fileTreeRootFile = setupJobTree(self.tempFileTree, self.jobTreeDir,
"parasol", depth=self.depth)
jobTreeCommand += " --rescueJobsFrequency 20"
#Run the job
parasolAndMasterKiller = ParasolAndMasterKiller()
parasolAndMasterKiller.start()
while True:
while True:
process = subprocess.Popen(jobTreeCommand, shell=True)
sts = os.waitpid(process.pid, 0)
if sts[1] == 0:
logger.info("The job tree master ended, with an okay exit value (using parasol)")
break
else:
logger.info("The job tree master ended with an error exit value, restarting: %i" % sts[1])
if checkEndStateOfJobTree(self.jobTreeDir): #Check the state of the job files
break
jobTreeCommand = "jobTreeRun --jobTree %s --logDebug" % self.jobTreeDir
checkFileTreeCounts(fileTreeRootFile)
os.system("rm -rf %s" % self.jobTreeDir)
parasolAndMasterKiller.stopKilling()
logger.info("Test done okay")
class TestCase(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.jobTreeDir = os.path.join(os.getcwd(), "testJobTree") #A directory for the job tree to be created in
self.tempFileTreeDir = os.path.join(os.getcwd(), "tempFileTree") #Ensures that file tree is visible
self.tempFileTree = TempFileTree(self.tempFileTreeDir) #A place to get temp files from
def tearDown(self):
unittest.TestCase.tearDown(self)
self.tempFileTree.destroyTempFiles()
system("rm -rf %s %s" % (self.jobTreeDir, self.tempFileTreeDir)) #Cleanup the job tree in case it hasn't already been cleaned up.
# only done in singleMachine for now. Experts can run manually on other systems if they choose
def dependenciesTest(self, batchSystem="singleMachine", furtherOptionsString=""):
def fn(tree, maxCpus, maxThreads, size, cpusPerJob, sleepTime):
system("rm -rf %s" % self.jobTreeDir)
logName = self.tempFileTree.getTempFile(suffix="_comblog.txt", makeDir=False)
commandLine = "jobTreeTest_Dependencies.py --jobTree %s --logFile %s --batchSystem '%s' --tree %s --maxCpus %s --maxThreads %s --size %s --cpusPerJob=%s --sleepTime %s %s" % \
(self.jobTreeDir, logName, batchSystem, tree, maxCpus, maxThreads, size, cpusPerJob, sleepTime, furtherOptionsString)
system(commandLine)
fn("comb", 10, 100, 100, 1, 10)
fn("comb", 200, 100, 100, 20, 10)
fn("fly", 10, 8, 100, 1, 10)
fn("fly", 10, 8, 100, 2, 10)
fn("balanced", 5, 10, 100, 1, 10)
fn("balanced", 5, 10, 100, 3, 10)
def testJobTree_dependencies_singleMachine(self):
self.dependenciesTest(batchSystem="singleMachine")
def testJobTree_dependencies_combined(self):
self.dependenciesTest(batchSystem="singleMachine", furtherOptionsString="--bigBatchSystem singleMachine --bigMemoryThreshold 1000000")
def testJobTree_dependencies_parasol(self):
return
if parasolIsInstalled():
self.dependenciesTest(batchSystem="parasol")
def testJobTree_dependencies_gridengine(self):
return
if gridEngineIsInstalled():
self.dependenciesTest(batchSystem="gridengine")
class TestCase(unittest.TestCase):
def setUp(self):
unittest.TestCase.setUp(self)
self.testNo = TestStatus.getTestSetup(1, 1, 2, 2)
self.depth = TestStatus.getTestSetup(1, 2, 3, 5)
self.jobTreeDir = os.path.join(os.getcwd(), "jobTree") #A directory for the job tree to be created in
self.tempFileTreeDir = os.path.join(os.getcwd(), "tempFileTree") #Ensures that file tree is visible
self.tempFileTree = TempFileTree(self.tempFileTreeDir) #A place to get temp files from
def tearDown(self):
unittest.TestCase.tearDown(self)
self.tempFileTree.destroyTempFiles()
system("rm -rf %s %s" % (self.jobTreeDir, self.tempFileTreeDir)) #Cleanup the job tree in case it hasn't already been cleaned up.
def testJobTree_SingleMachine(self):
testJobTree(self.testNo, self.depth, self.tempFileTree, self.jobTreeDir, "singleMachine")
def testJobTree_Parasol(self):
if parasolIsInstalled():
testJobTree(self.testNo, self.depth, self.tempFileTree, self.jobTreeDir, "parasol")
def testJobTree_gridengine(self):
if gridEngineIsInstalled():
testJobTree(self.testNo, self.depth, self.tempFileTree, self.jobTreeDir, "gridengine")
def testJobTree_dependencies(self):
commandLine = "jobTreeTest_Dependencies.py --jobTree %s --tree comb --maxThreads 100" % self.jobTreeDir
os.system("rm -rf %s" % self.jobTreeDir)
system(commandLine)
commandLine = "jobTreeTest_Dependencies.py --jobTree %s --tree fly --maxThreads 100" % self.jobTreeDir
os.system("rm -rf %s" % self.jobTreeDir)
system(commandLine)
os.system("rm -rf %s" % self.jobTreeDir)
commandLine = "jobTreeTest_Dependencies.py --jobTree %s --tree balanced --maxThreads 100" % self.jobTreeDir
system(commandLine)
def testTempFileTree(self):
for test in range(100): #self.testNo):
levels = random.choice(range(1, 4))
fileNo = random.choice(range(1, 6))
maxTempFiles = int(math.pow(fileNo, levels))
print("Got %s levels, %s fileNo and %s maxTempFiles" %
(levels, fileNo, maxTempFiles))
tempFileTreeRootDir = os.path.join(self.tempDir,
getRandomAlphaNumericString())
tempFileTree = TempFileTree(tempFileTreeRootDir, fileNo, levels)
tempFiles = []
tempDirs = []
#Check we can mac number of temp files.
for i in range(maxTempFiles):
if random.random() > 0.5:
tempFile = tempFileTree.getTempFile()
assert os.path.isfile(tempFile)
tempFiles.append(tempFile)
else:
tempFile = tempFileTree.getTempDirectory()
assert os.path.isdir(tempFile)
tempDirs.append(tempFile)
#Check assertion is created
try:
tempFileTree.getTempFile()
assert False
except RuntimeError:
logger.debug("Got expected error message")
#Now remove a few temp files
while random.random() > 0.1 and len(tempFiles) > 0:
tempFile = tempFiles.pop()
assert os.path.isfile(tempFile)
tempFileTree.destroyTempFile(tempFile)
assert not os.path.isfile(tempFile)
#Now remove a few temp dirs
while random.random() > 0.1 and len(tempDirs) > 0:
tempDir = tempDirs.pop()
assert os.path.isdir(tempDir)
tempFileTree.destroyTempDir(tempDir)
assert not os.path.isdir(tempDir)
#Check temp files is okay
set(tempFileTree.listFiles()) == set(tempFiles + tempDirs)
#Either remove all the temp files or just destroy the whole thing
if random.random() > 0.5:
#Remove all temp files and check thing is empty.
for tempFile in tempFiles:
tempFileTree.destroyTempFile(tempFile)
for tempDir in tempDirs:
tempFileTree.destroyTempDir(tempDir)
os.remove(os.path.join(tempFileTreeRootDir, "lock"))
os.rmdir(tempFileTreeRootDir)
else:
tempFileTree.destroyTempFiles()
assert not os.path.isdir(tempFileTreeRootDir)
