def testCactus_Chromosomes(self):
runWorkflow_multipleExamples(getCactusInputs_chromosomeX,
testRestrictions=(TestStatus.TEST_VERY_LONG,),
batchSystem=self.batchSystem, buildJobTreeStats=True,
progressive=True,
configFile=self.configFile,
cactusWorkflowFunction=self.progressiveFunction)
def testCactus_randomWithConstraints(self):
runWorkflow_multipleExamples(getCactusInputs_randomWithConstraints,
testNumber=1,
testRestrictions=(TestStatus.TEST_SHORT,),
buildAvgs=True, buildReference=True,
batchSystem=self.batchSystem, buildToilStats=True,
useConstraints=True)
def runCactus_Random(self, matchingAlgorithm):
configFile = getConfigFile(matchingAlgorithm)
runWorkflow_multipleExamples(self.id(),
getCactusInputs_random,
testNumber=TestStatus.getTestSetup(),
configFile=configFile)
os.remove(configFile)
def testCactus_Blanchette(self, matchingAlgorithm):
configFile = getConfigFile(matchingAlgorithm)
runWorkflow_multipleExamples(getCactusInputs_blanchette,
testRestrictions=(TestStatus.TEST_SHORT,), inverseTestRestrictions=True,
buildReference=True,
configFile=configFile)
os.remove(configFile)
def testCactus_Random(self):
runWorkflow_multipleExamples(getCactusInputs_random,
testNumber=1,
batchSystem=self.batchSystem, buildToilStats=True,
progressive=True,
configFile=self.configFile,
cactusWorkflowFunction=self.progressiveFunction)
def testEvolver_Mammals_Large(self):
inputDir = os.path.join(TestStatus.getPathToDataSets(), "evolver", "mammals", "large")
mammalSequences = ("simCow.masked.fa", "simDog.masked.fa", "simHuman.masked.fa", "simMouse.masked.fa", "simRat.masked.fa")
runWorkflow_multipleExamples(self.id(),
lambda regionNumber=0, tempDir=None : getInputs(inputDir, mammalSequences),
batchSystem=self.batchSystem,
buildToilStats=True)
def testCactus_Random(self, matchingAlgorithm):
configFile = getConfigFile(matchingAlgorithm)
runWorkflow_multipleExamples(getCactusInputs_random,
testNumber=TestStatus.getTestSetup(),
buildReference=True,
configFile=configFile)
os.remove(configFile)
def testCactus_Random(self, matchingAlgorithm):
configFile = getConfigFile(matchingAlgorithm)
runWorkflow_multipleExamples(getCactusInputs_random,
testNumber=TestStatus.getTestSetup(),
buildReference=True,
configFile=configFile)
os.remove(configFile)
def testEvolver_Primates_Large(self):
inputDir = os.path.join(TestStatus.getPathToDataSets(), "evolver", "primates", "large")
primateSequences = ("simChimp.masked.fa", "simGorilla.masked.fa", "simHuman.masked.fa", "simOrang.masked.fa")
runWorkflow_multipleExamples(self.id(),
lambda regionNumber=0, tempDir=None : getInputs(inputDir, primateSequences),
batchSystem=self.batchSystem,
buildToilStats=True)
def testCactus_encode(self):
runWorkflow_multipleExamples(self.id(),
getCactusInputs_encode,
testNumber=1,
buildAvgs=True,
batchSystem=self.batchSystem,
buildToilStats=True)
def testCactus_randomWithConstraints(self):
runWorkflow_multipleExamples(getCactusInputs_randomWithConstraints,
testNumber=5,
testRestrictions=(TestStatus.TEST_SHORT,),
buildAvgs=True, buildReference=True,
batchSystem=self.batchSystem, buildJobTreeStats=True,
useConstraints=True)
def testCactus_Blanchette(self, matchingAlgorithm):
configFile = getConfigFile(matchingAlgorithm)
runWorkflow_multipleExamples(getCactusInputs_blanchette,
testRestrictions=(TestStatus.TEST_SHORT, ),
inverseTestRestrictions=True,
buildReference=True,
configFile=configFile)
os.remove(configFile)
def testCactus_Encode(self):
runWorkflow_multipleExamples(getCactusInputs_encode,
testNumber=1,
testRestrictions=(TestStatus.TEST_LONG,),
batchSystem=self.batchSystem, buildJobTreeStats=True,
progressive=True,
configFile=self.configFile,
cactusWorkflowFunction=self.progressiveFunction)
def testCactus_encode(self):
runWorkflow_multipleExamples(getCactusInputs_encode,
testNumber=1,
testRestrictions=(TestStatus.TEST_LONG, ),
buildAvgs=True,
buildReference=True,
batchSystem=self.batchSystem,
buildToilStats=True)
def testCactus_ensureFunkyHeaderNamesArentMangled(self):
"""Ensure header names with characters like "|", " " aren't mangled."""
runWorkflow_multipleExamples(getCactusInputs_funkyHeaderNames,
testNumber=1,
batchSystem=self.batchSystem, buildToilStats=True,
progressive=True,
configFile=self.configFile,
cactusWorkflowFunction=self.progressiveFunction)
def testCactus_Random(self):
runWorkflow_multipleExamples(getCactusInputs_random,
testNumber=2,
testRestrictions=(TestStatus.TEST_SHORT,),
batchSystem=self.batchSystem, buildJobTreeStats=True,
progressive=True,
configFile=self.configFile,
cactusWorkflowFunction=self.progressiveFunction)
def testCactusRecursiveHalGenerator_Blanchette(self):
runWorkflow_multipleExamples(
getCactusInputs_blanchette,
testRestrictions=(TestStatus.TEST_SHORT, ),
inverseTestRestrictions=True,
buildReference=True,
buildHal=True,
buildFasta=True)
def testCactus_Blanchette(self):
runWorkflow_multipleExamples(getCactusInputs_blanchette,
testNumber=1,
testRestrictions=(TestStatus.TEST_MEDIUM,),
batchSystem=self.batchSystem, buildToilStats=True,
progressive=True,
configFile=self.configFile,
cactusWorkflowFunction=self.progressiveFunction)
def testCactus_Chromosomes(self):
runWorkflow_multipleExamples(
self.id(),
getCactusInputs_chromosomeX,
batchSystem=self.batchSystem,
buildToilStats=True,
progressive=True,
configFile=self.configFile,
cactusWorkflowFunction=self.progressiveFunction)
def testEvolver_Mammals_Loci1(self):
if "SON_TRACE_DATASETS" not in os.environ:
return
inputDir = os.path.join(TestStatus.getPathToDataSets(), "evolver", "mammals", "loci1")
mammalSequences = ("simCow.chr6", "simDog.chr6", "simHuman.chr6", "simMouse.chr6", "simRat.chr6")
runWorkflow_multipleExamples(lambda regionNumber=0, tempDir=None : getInputs(inputDir, mammalSequences),
testRestrictions=(TestStatus.TEST_MEDIUM,),
batchSystem=self.batchSystem,
buildToilStats=True)
def testCactus_Chromosomes(self):
if "SON_TRACE_DATASETS" not in os.environ:
return
runWorkflow_multipleExamples(getCactusInputs_chromosomeX,
testRestrictions=(TestStatus.TEST_VERY_LONG,),
batchSystem=self.batchSystem, buildToilStats=True,
progressive=True,
configFile=self.configFile,
cactusWorkflowFunction=self.progressiveFunction)
def testEvolver_Primates_Large(self):
if "SON_TRACE_DATASETS" not in os.environ:
return
inputDir = os.path.join(TestStatus.getPathToDataSets(), "evolver", "primates", "large")
primateSequences = ("simChimp.masked.fa", "simGorilla.masked.fa", "simHuman.masked.fa", "simOrang.masked.fa")
runWorkflow_multipleExamples(lambda regionNumber=0, tempDir=None : getInputs(inputDir, primateSequences),
testRestrictions=(TestStatus.TEST_VERY_LONG,),
batchSystem=self.batchSystem,
buildJobTreeStats=True)
def testEvolver_Mammals_Large(self):
if "SON_TRACE_DATASETS" not in os.environ:
return
inputDir = os.path.join(TestStatus.getPathToDataSets(), "evolver", "mammals", "large")
mammalSequences = ("simCow.masked.fa", "simDog.masked.fa", "simHuman.masked.fa", "simMouse.masked.fa", "simRat.masked.fa")
runWorkflow_multipleExamples(lambda regionNumber=0, tempDir=None : getInputs(inputDir, mammalSequences),
testRestrictions=(TestStatus.TEST_VERY_LONG,),
batchSystem=self.batchSystem,
buildToilStats=True)
def testEvolver_Mammals_Large(self):
if "SON_TRACE_DATASETS" not in os.environ:
return
inputDir = os.path.join(TestStatus.getPathToDataSets(), "evolver", "mammals", "large")
mammalSequences = ("simCow.masked.fa", "simDog.masked.fa", "simHuman.masked.fa", "simMouse.masked.fa", "simRat.masked.fa")
runWorkflow_multipleExamples(lambda regionNumber=0, tempDir=None : getInputs(inputDir, mammalSequences),
testRestrictions=(TestStatus.TEST_VERY_LONG,),
batchSystem=self.batchSystem,
buildJobTreeStats=True)
def testEvolver_Primates_Loci1(self):
if "SON_TRACE_DATASETS" not in os.environ:
return
inputDir = os.path.join(TestStatus.getPathToDataSets(), "evolver", "primates", "loci1")
primateSequences = ("simChimp.chr6", "simGorilla.chr6", "simHuman.chr6", "simOrang.chr6")
runWorkflow_multipleExamples(lambda regionNumber=0, tempDir=None : getInputs(inputDir, primateSequences),
testRestrictions=(TestStatus.TEST_SHORT,),
batchSystem=self.batchSystem,
buildToilStats=True)
def testEvolver_Primates_Large(self):
if "SON_TRACE_DATASETS" not in os.environ:
return
inputDir = os.path.join(TestStatus.getPathToDataSets(), "evolver", "primates", "large")
primateSequences = ("simChimp.masked.fa", "simGorilla.masked.fa", "simHuman.masked.fa", "simOrang.masked.fa")
runWorkflow_multipleExamples(lambda regionNumber=0, tempDir=None : getInputs(inputDir, primateSequences),
testRestrictions=(TestStatus.TEST_VERY_LONG,),
batchSystem=self.batchSystem,
buildToilStats=True)
def testCactus_Blanchette(self):
runWorkflow_multipleExamples(
getCactusInputs_blanchette,
testNumber=1,
testRestrictions=(TestStatus.TEST_MEDIUM, ),
batchSystem=self.batchSystem,
buildToilStats=True,
progressive=True,
configFile=self.configFile,
cactusWorkflowFunction=self.progressiveFunction)
def testCactus_Random_UseSubtreeRoot(self):
"""Tests that cactus doesn't crash when aligning a subtree of a larger
species tree."""
runWorkflow_multipleExamples(getCactusInputs_random,
testNumber=2,
testRestrictions=(TestStatus.TEST_SHORT,),
batchSystem=self.batchSystem, buildJobTreeStats=True,
progressive=True,
configFile=self.configFile,
cactusWorkflowFunction=self.progressiveWithSubtreeRootFunction)
def testCactus_Blanchette_UseOutgroupAndDoSelfAlignment(self):
self.useOutgroup = True
self.doSelfAlignment = True
runWorkflow_multipleExamples(getCactusInputs_blanchette,
testNumber=1,
testRestrictions=(TestStatus.TEST_MEDIUM,),
batchSystem=self.batchSystem, buildJobTreeStats=True,
progressive=True,
configFile=self.configFile,
cactusWorkflowFunction=self.progressiveFunction)
def testCactus_Blanchette(self):
runWorkflow_multipleExamples(
self.id(),
getCactusInputs_blanchette,
testNumber=1,
batchSystem=self.batchSystem,
buildToilStats=True,
progressive=True,
configFile=self.configFile,
cactusWorkflowFunction=self.progressiveFunction)
def testCactus_Random_UseSubtreeRoot(self):
"""Tests that cactus doesn't crash when aligning a subtree of a larger
species tree."""
getBigSpeciesTree = lambda regionNumber=0,tempDir=None: getCactusInputs_random(regionNumber, tempDir, treeLeafNumber=5)
runWorkflow_multipleExamples(getBigSpeciesTree,
testNumber=1,
batchSystem=self.batchSystem, buildToilStats=True,
progressive=True,
configFile=self.configFile,
cactusWorkflowFunction=self.progressiveWithSubtreeRootFunction)
def testCactus_Chromosomes(self):
if "SON_TRACE_DATASETS" not in os.environ:
return
runWorkflow_multipleExamples(
getCactusInputs_chromosomeX,
testRestrictions=(TestStatus.TEST_VERY_LONG, ),
batchSystem=self.batchSystem,
buildToilStats=True,
progressive=True,
configFile=self.configFile,
cactusWorkflowFunction=self.progressiveFunction)
def testCactus_ensureFunkyHeaderNamesArentMangled(self):
"""Ensure header names with characters like "|", " " aren't mangled."""
if "SON_TRACE_DATASETS" not in os.environ:
return
runWorkflow_multipleExamples(getCactusInputs_funkyHeaderNames,
testNumber=1,
testRestrictions=(TestStatus.TEST_SHORT,),
batchSystem=self.batchSystem, buildToilStats=True,
progressive=True,
configFile=self.configFile,
cactusWorkflowFunction=self.progressiveFunction)
def testCactus_random(self):
# gets "Couldn't find reference event reference" from cactus_reference
# The error means that it is getting told to calculate the ancestral
# sequence for the species "reference" but that species doesn't
# actually exist in the tree
runWorkflow_multipleExamples(self.id(),
getCactusInputs_random,
testNumber=1,
buildAvgs=True,
batchSystem=self.batchSystem,
buildToilStats=True)
def testCactus_ensureFunkyHeaderNamesArentMangled(self):
"""Ensure header names with characters like "|", " " aren't mangled."""
if "SON_TRACE_DATASETS" not in os.environ:
return
runWorkflow_multipleExamples(
getCactusInputs_funkyHeaderNames,
testNumber=1,
testRestrictions=(TestStatus.TEST_SHORT, ),
batchSystem=self.batchSystem,
buildToilStats=True,
progressive=True,
configFile=self.configFile,
cactusWorkflowFunction=self.progressiveFunction)
def testCactus_Random(self):
# TODO: this doesn't actually need the test data, but this is
# being used as a signal that the tester doesn't want to run
# the long tests. The tests should be refactored soon.
if "SON_TRACE_DATASETS" not in os.environ:
return
runWorkflow_multipleExamples(getCactusInputs_random,
testNumber=1,
testRestrictions=(TestStatus.TEST_SHORT,),
batchSystem=self.batchSystem, buildToilStats=True,
progressive=True,
configFile=self.configFile,
cactusWorkflowFunction=self.progressiveFunction)
def testCactus_Random(self):
# TODO: this doesn't actually need the test data, but this is
# being used as a signal that the tester doesn't want to run
# the long tests. The tests should be refactored soon.
if "SON_TRACE_DATASETS" not in os.environ:
return
runWorkflow_multipleExamples(
getCactusInputs_random,
testNumber=1,
testRestrictions=(TestStatus.TEST_SHORT, ),
batchSystem=self.batchSystem,
buildToilStats=True,
progressive=True,
configFile=self.configFile,
cactusWorkflowFunction=self.progressiveFunction)
def testCactus_splitBarJobs(self):
"""Exercise the code paths in bar that only occur on large jobs."""
# Modify the bar node in the config file so that
# cactus_workflow will split bar jobs even on this small
# example
tempConfigFile = getTempFile()
tempConfigTree = ET.parse(self.configFile)
tempConfigNode = tempConfigTree.getroot()
tempConfigNode.find("bar").find("CactusBarWrapper").set("maxFlowerGroupSize", "10")
tempConfigNode.find("bar").set("veryLargeEndSize", "0")
tempConfigNode.find("bar").set("largeEndSize", "0")
tempConfigTree.write(tempConfigFile)
runWorkflow_multipleExamples(getCactusInputs_blanchette,
testNumber=1,
testRestrictions=(TestStatus.TEST_LONG,),
batchSystem=self.batchSystem,
configFile=tempConfigFile, buildJobTreeStats=True)
os.remove(tempConfigFile)
def testCactus_Random_UseSubtreeRoot(self):
# TODO: this doesn't actually need the test data, but this is
# being used as a signal that the tester doesn't want to run
# the long tests. The tests should be refactored soon.
if "SON_TRACE_DATASETS" not in os.environ:
return
"""Tests that cactus doesn't crash when aligning a subtree of a larger
species tree."""
getBigSpeciesTree = lambda regionNumber=0, tempDir=None: getCactusInputs_random(
regionNumber, tempDir, treeLeafNumber=5)
runWorkflow_multipleExamples(
getBigSpeciesTree,
testNumber=1,
testRestrictions=(TestStatus.TEST_SHORT, ),
batchSystem=self.batchSystem,
buildToilStats=True,
progressive=True,
configFile=self.configFile,
cactusWorkflowFunction=self.progressiveWithSubtreeRootFunction)
def testCactus_splitBarJobs(self):
"""Exercise the code paths in bar that only occur on large jobs."""
# Modify the bar node in the config file so that
# cactus_workflow will split bar jobs even on this small
# example
initialiseGlobalDatabaseConf('<st_kv_database_conf type="kyoto_tycoon"><kyoto_tycoon in_memory="1" port="1978" snapshot="0"/></st_kv_database_conf>')
tempConfigFile = getTempFile()
tempConfigTree = ET.parse(self.configFile)
tempConfigNode = tempConfigTree.getroot()
tempConfigNode.find("bar").find("CactusBarWrapper").set("maxFlowerGroupSize", "10")
tempConfigNode.find("bar").find("CactusBarWrapperLarge").set("maxFlowerGroupSize", "10")
tempConfigNode.find("bar").set("veryLargeEndSize", "20")
tempConfigNode.find("bar").set("largeEndSize", "10")
tempConfigNode.find("bar").set("bandingLimit", "5")
tempConfigTree.write(tempConfigFile)
runWorkflow_multipleExamples(getCactusInputs_random,
testNumber=1,
batchSystem=self.batchSystem,
configFile=tempConfigFile)
os.remove(tempConfigFile)
def testCactus_splitBarJobs(self):
"""Exercise the code paths in bar that only occur on large jobs."""
# Modify the bar node in the config file so that
# cactus_workflow will split bar jobs even on this small
# example
initialiseGlobalDatabaseConf('<st_kv_database_conf type="kyoto_tycoon"><kyoto_tycoon in_memory="1" port="1978" snapshot="0"/></st_kv_database_conf>')
tempConfigFile = getTempFile()
tempConfigTree = ET.parse(self.configFile)
tempConfigNode = tempConfigTree.getroot()
tempConfigNode.find("bar").find("CactusBarWrapper").set("maxFlowerGroupSize", "10")
tempConfigNode.find("bar").find("CactusBarWrapperLarge").set("maxFlowerGroupSize", "10")
tempConfigNode.find("bar").set("veryLargeEndSize", "20")
tempConfigNode.find("bar").set("largeEndSize", "10")
tempConfigNode.find("bar").set("bandingLimit", "5")
tempConfigTree.write(tempConfigFile)
runWorkflow_multipleExamples(getCactusInputs_random,
testNumber=1,
batchSystem=self.batchSystem,
configFile=tempConfigFile)
os.remove(tempConfigFile)
def testCactus_splitBarJobs(self):
"""Exercise the code paths in bar that only occur on large jobs."""
# Modify the bar node in the config file so that
# cactus_workflow will split bar jobs even on this small
# example
tempConfigFile = getTempFile()
tempConfigTree = ET.parse(self.configFile)
tempConfigNode = tempConfigTree.getroot()
tempConfigNode.find("bar").find("CactusBarWrapper").set(
"maxFlowerGroupSize", "10")
tempConfigNode.find("bar").find("CactusBarWrapperLarge").set(
"maxFlowerGroupSize", "10")
tempConfigNode.find("bar").set("veryLargeEndSize", "20")
tempConfigNode.find("bar").set("largeEndSize", "10")
tempConfigNode.find("bar").set("bandingLimit", "5")
tempConfigTree.write(tempConfigFile)
runWorkflow_multipleExamples(self.id(),
getCactusInputs_random,
testNumber=1,
batchSystem=self.batchSystem,
configFile=tempConfigFile)
os.remove(tempConfigFile)
def testCactus_Blanchette(self):
runWorkflow_multipleExamples(self.id(),
getCactusInputs_blanchette,
buildAvgs=True)
def testCactus_chromosomes(self):
runWorkflow_multipleExamples(getCactusInputs_chromosomeX,
testRestrictions=(TestStatus.TEST_VERY_LONG,),
batchSystem=self.batchSystem, buildJobTreeStats=True)
def testCactus_encode(self):
runWorkflow_multipleExamples(getCactusInputs_encode,
testNumber=1,
testRestrictions=(TestStatus.TEST_LONG,),
buildAvgs=True, buildReference=True,
batchSystem=self.batchSystem, buildJobTreeStats=True)
def testCactusCore_Blanchette(self):
runWorkflow_multipleExamples(
getCactusInputs_blanchette,
testRestrictions=(TestStatus.TEST_SHORT, ),
inverseTestRestrictions=True)
def testCactusCore_Blanchette(self):
runWorkflow_multipleExamples(getCactusInputs_blanchette,
testRestrictions=(TestStatus.TEST_SHORT,), inverseTestRestrictions=True)
def testCactusCore_Random(self):
for test in xrange(TestStatus.getTestSetup()):
randomConfigFile=getRandomConfigFile()
runWorkflow_multipleExamples(getCactusInputs_random,
configFile=randomConfigFile)
os.remove(randomConfigFile)
def testCactus_chromosomes(self):
runWorkflow_multipleExamples(self.id(),
getCactusInputs_chromosomeX,
batchSystem=self.batchSystem,
buildToilStats=True)
def testCactus_blanchette(self):
runWorkflow_multipleExamples(getCactusInputs_blanchette,
testNumber=1,
buildAvgs=True,
batchSystem=self.batchSystem, buildToilStats=True)
def testCactus_Random(self):
runWorkflow_multipleExamples(self.id(),
getCactusInputs_random,
testNumber=TestStatus.getTestSetup(),
buildAvgs=True)
def testCactusNormalisation_Random(self):
runWorkflow_multipleExamples(getCactusInputs_random,
testNumber=TestStatus.getTestSetup())
def testCactusCore_Random(self):
for test in xrange(TestStatus.getTestSetup()):
randomConfigFile = getRandomConfigFile()
runWorkflow_multipleExamples(getCactusInputs_random,
configFile=randomConfigFile)
os.remove(randomConfigFile)