fastlmm.association.tests.test_gwas.getTestSuite(),
fastlmm.util.testdistributable.getTestSuite(),
fastlmm.feature_selection.test.getTestSuite(),
])
suites.debug
if True: #Standard test run
r = unittest.TextTestRunner(failfast=False)
r.run(suites)
else: #Cluster test run
task_count = 150
runner = HPC(
task_count,
'RR1-N13-09-H44',
r'\\msr-arrays\Scratch\msr-pool\Scratch_Storage6\Redmond',
remote_python_parent=
r"\\msr-arrays\Scratch\msr-pool\Scratch_Storage6\REDMOND\carlk\Source\carlk\july_7_14\pythonpath",
update_remote_python_parent=True,
min=150,
priority="AboveNormal",
mkl_num_threads=1)
runner = Local()
runner = LocalMultiProc(taskcount=20, mkl_num_threads=5)
#runner = LocalInParts(1,2,mkl_num_threads=1) # For debugging the cluster runs
#runner = Hadoop2(100, mapmemory=8*1024, reducememory=8*1024, mkl_num_threads=1, queue="default")
distributable_test = DistributableTest(suites, "temp_test")
print runner.run(distributable_test)
debian_count = len(os.listdir('expected-debian'))
if debian_count > 0:
logging.warn(
"The tests contain {0} expected-results files that differ between Windows and Debian"
if __name__ == '__main__':
from fastlmm.association.tests.testepistasis import TestEpistasis
suites = unittest.TestSuite([getTestSuite()])
if False: #Standard test run
r = unittest.TextTestRunner(failfast=False)
r.run(suites)
else: #Cluster test run
from fastlmm.util.distributabletest import DistributableTest
runner = HPC(
10,
'RR1-N13-09-H44',
r'\\msr-arrays\Scratch\msr-pool\Scratch_Storage4\Redmond',
remote_python_parent=
r"\\msr-arrays\Scratch\msr-pool\Scratch_Storage4\REDMOND\carlk\Source\carlk\july_7_14\tests\runs\2014-07-24_15_02_02_554725991686\pythonpath",
update_remote_python_parent=True,
priority="AboveNormal",
mkl_num_threads=1)
runner = Local()
#runner = LocalMultiProc(taskcount=20,mkl_num_threads=5)
#runner = LocalInParts(1,2,mkl_num_threads=1) # For debugging the cluster runs
#runner = Hadoop(100, mapmemory=8*1024, reducememory=8*1024, mkl_num_threads=1, queue="default")
distributable_test = DistributableTest(suites, "temp_test")
print runner.run(distributable_test)
logging.info("done with testing")
if True: #Standard test run
r = unittest.TextTestRunner(failfast=False)
r.run(suites)
else: #Cluster test run
logging.basicConfig(level=logging.INFO)
from fastlmm.util.distributabletest import DistributableTest
remote_python_parent = r"\\GCR\Scratch\RR1\escience\carlk\data\carlk\pythonpath06292016"
runner = HPC(
2,
'GCR',
r"\\GCR\Scratch\RR1\escience",
remote_python_parent=remote_python_parent,
unit='node', #core, socket, node
update_remote_python_parent=True,
template="Preemptable",
priority="Lowest",
nodegroups="Preemptable",
#excluded_nodes=['gcrcn0231'],
runtime="0:11:0", # day:hour:min
max=10)
#runner = Local()
#runner = LocalMultiProc(taskcount=2,mkl_num_threads=5,just_one_process=False)
#runner = LocalInParts(0,2,mkl_num_threads=1) # For debugging the cluster runs
#runner = Hadoop(100, mapmemory=8*1024, reducememory=8*1024, mkl_num_threads=1, queue="default")
distributable_test = DistributableTest(suites, "temp_test")
print runner.run(distributable_test)
logging.info("done with testing")
if True: #Standard test run
r = unittest.TextTestRunner(failfast=False)
r.run(suites)
else: #Cluster test run
logging.basicConfig(level=logging.INFO)
from fastlmm.util.distributabletest import DistributableTest
remote_python_parent=r"\\GCR\Scratch\RR1\escience\carlk\data\carlk\pythonpath06292016"
runner = HPC(2, 'GCR',r"\\GCR\Scratch\RR1\escience",
remote_python_parent=remote_python_parent,
unit='node', #core, socket, node
update_remote_python_parent=True,
template="Preemptable",
priority="Lowest",
nodegroups="Preemptable",
#excluded_nodes=['gcrcn0231'],
runtime="0:11:0", # day:hour:min
max = 10
)
#runner = Local()
#runner = LocalMultiProc(taskcount=2,mkl_num_threads=5,just_one_process=False)
#runner = LocalInParts(0,2,mkl_num_threads=1) # For debugging the cluster runs
#runner = Hadoop(100, mapmemory=8*1024, reducememory=8*1024, mkl_num_threads=1, queue="default")
distributable_test = DistributableTest(suites,"temp_test")
print runner.run(distributable_test)
logging.info("done with testing")
def mf_to_runner_function(mf):
excluded_nodes = [
] #'GCRCM07B20','GCRCM11B05','GCRCM10B06','GCRCM02B07']#'GCRCM02B11','GCRCM03B07'] #'GCRCM22B06','GCRCN0383','GCRCM02B07','GCRCN0179','GCRCM37B13','GCRCN0376','GCRCN0456']#'gcrcn0231']#"MSR-HDP-DN0316","MSR-HDP-DN0321","MSR-HDP-DN0336","MSR-HDP-DN0377","MSR-HDP-DN0378","MSR-HDP-DN0314","MSR-HDP-DN0335","MSRQC073","MSRQC002","MSRQC015"]
remote_python_parent = r"\\GCR\Scratch\RR1\escience\carlk\data\carlk\pythonpath10262016"
clean_up = False
if mf == "debug":
runner_function = lambda ignore: LocalInParts(
215,
215,
mkl_num_threads=20,
result_file="result.p",
run_dir=r"C:\deldir\test\outputx")
elif mf == "local":
runner_function = lambda ignore: Local()
elif mf == "local1":
runner_function = lambda ignore: Local(1)
elif mf == "lmp":
runner_function = lambda ignore: LocalMultiProc(22, 5)
elif mf == "lmp4":
runner_function = lambda ignore: LocalMultiProc(4, 5)
elif mf == "lmpl":
runner_function = lambda taskcount: LocalMultiProc(
taskcount, taskcount, just_one_process=True)
elif mf == "nodeP":
runner_function = lambda taskcount: HPC(
min(taskcount, 30100),
'GCR',
r"\\GCR\Scratch\RR1\escience",
remote_python_parent=remote_python_parent,
unit='node', #core, socket, node
update_remote_python_parent=True,
template="Preemptable",
priority="Lowest",
excluded_nodes=excluded_nodes,
#mkl_num_threads=20,
nodegroups="Preemptable",
runtime="0:11:0", # day:hour:min
#min = 10 #max(1,min(taskcount,110)//20)
#max = min(taskcount,500),
clean_up=clean_up,
)
elif mf == "nodeP99":
runner_function = lambda taskcount: HPC(
min(taskcount, 30100),
'GCR',
r"\\GCR\Scratch\RR1\escience",
remote_python_parent=remote_python_parent,
unit='node', #core, socket, node
update_remote_python_parent=True,
template="Preemptable",
priority="Lowest",
excluded_nodes=excluded_nodes,
#mkl_num_threads=20,
nodegroups="Preemptable,B99",
runtime="0:11:0", # day:hour:min
#min = 10 #max(1,min(taskcount,110)//20)
#max = min(taskcount,500),
clean_up=clean_up,
)
elif mf == "nodeL99":
runner_function = lambda taskcount: HPC(
min(taskcount, 30100),
'GCR',
r"\\GCR\Scratch\RR1\escience",
remote_python_parent=remote_python_parent,
unit='node', #core, socket, node
update_remote_python_parent=True,
template="LongRunQ",
priority="Lowest",
excluded_nodes=excluded_nodes,
#mkl_num_threads=20,
nodegroups="LongRunQ,B99",
runtime="11:0:0", # day:hour:min
#min = 10 #max(1,min(taskcount,110)//20)
#max = min(taskcount,500),
clean_up=clean_up,
)
elif mf == "socketP":
runner_function = lambda taskcount: HPC(
min(taskcount, 30100),
'GCR',
r"\\GCR\Scratch\RR1\escience",
remote_python_parent=remote_python_parent,
unit='socket', #core, socket, node
update_remote_python_parent=True,
template="Preemptable",
priority="Lowest",
excluded_nodes=excluded_nodes,
mkl_num_threads=10,
nodegroups="Preemptable",
runtime="0:11:0", # day:hour:min
#min = max(1,min(taskcount,110)//20),
clean_up=clean_up,
)
elif mf == "coreP":
runner_function = lambda taskcount: HPC(
min(taskcount, 1000),
'GCR',
r"\\GCR\Scratch\RR1\escience",
remote_python_parent=remote_python_parent,
unit='core', #core, socket, node
update_remote_python_parent=True,
template="Preemptable",
priority="Lowest",
excluded_nodes=excluded_nodes,
mkl_num_threads=1,
runtime="0:11:0", # day:hour:min
nodegroups="Preemptable",
#min = min(taskcount,1100)
min=1,
max=200 * 20,
clean_up=clean_up,
)
elif mf == "coreP99":
runner_function = lambda taskcount: HPC(
min(taskcount, 1000),
'GCR',
r"\\GCR\Scratch\RR1\escience",
remote_python_parent=remote_python_parent,
unit='core', #core, socket, node
update_remote_python_parent=True,
template="Preemptable",
priority="Lowest",
excluded_nodes=excluded_nodes,
mkl_num_threads=1,
runtime="0:11:0", # day:hour:min
nodegroups="Preemptable,B99",
#min = min(taskcount,1100)
min=1,
max=200 * 20,
clean_up=clean_up,
)
elif mf == "coreAz":
runner_function = lambda taskcount: HPC(
min(taskcount, 1000),
'GCR',
r"\\GCR\Scratch\AZ-USCentral\escience",
remote_python_parent=
r"\\GCR\Scratch\AZ-USCentral\escience\carlk\data\carlk\pythonpath",
unit='core', #core, socket, node
update_remote_python_parent=True,
template="Azure IaaS USCentral",
mkl_num_threads=1,
runtime="0:8:0", # day:hour:min,
clean_up=clean_up,
)
elif mf == "nodeE":
runner_function = lambda taskcount: HPC(
min(taskcount, 10100),
'GCR',
r"\\GCR\Scratch\RR1\escience",
remote_python_parent=remote_python_parent,
unit='node', #core, socket, node
update_remote_python_parent=True,
template="ExpressQ",
priority="Normal",
#node_local = False,
#mkl_num_threads=20,
runtime="0:4:0", # day:hour:min
#min = min(taskcount,100),
clean_up=clean_up,
)
elif mf == "50tasks":
runner_function = lambda taskcount: HPC(
50,
'GCR',
r"\\GCR\Scratch\RR1\escience",
remote_python_parent=remote_python_parent,
unit='node', #core, socket, node
update_remote_python_parent=True,
template="ExpressQ",
priority="Normal",
#mkl_num_threads=20,
runtime="0:4:0", # day:hour:min
#min = min(taskcount,100),
clean_up=clean_up,
)
elif mf == "coreE":
runner_function = lambda taskcount: HPC(
min(taskcount, 10100),
'GCR',
r"\\GCR\Scratch\RR1\escience",
remote_python_parent=remote_python_parent,
unit='core', #core, socket, node
update_remote_python_parent=True,
template="ExpressQ",
priority="Normal",
mkl_num_threads=1,
runtime="0:4:0", # day:hour:min
#min = min(taskcount,100),
clean_up=clean_up,
)
elif mf == "nodeA":
runner_function = lambda taskcount: HPC(
min(taskcount, 30100),
'GCR',
r"\\GCR\Scratch\RR1\escience",
remote_python_parent=remote_python_parent,
unit='node', #core, socket, node
update_remote_python_parent=True,
template="Admin Template",
clean_up=clean_up,
)
elif mf == "socketA":
runner_function = lambda taskcount: HPC(
min(taskcount, 30100),
'GCR',
r"\\GCR\Scratch\RR1\escience",
remote_python_parent=remote_python_parent,
unit='socket', #core, socket, node
update_remote_python_parent=True,
template="Admin Template",
clean_up=clean_up,
)
elif mf == "coreA":
runner_function = lambda taskcount: HPC(
min(taskcount, 30100),
'GCR',
r"\\GCR\Scratch\RR1\escience",
remote_python_parent=remote_python_parent,
unit='core', #core, socket, node
update_remote_python_parent=True,
template="Admin Template",
clean_up=clean_up,
)
elif mf == "nodeH":
runner_function = lambda taskcount: Hadoop2(min(taskcount, 100000),
mapmemory=58 * 1024,
reducememory=8 * 1024,
min_alloc=2048,
xmx=3072,
mkl_num_threads=14,
queue="shared",
skipdatacheck=True,
skipsourcecheck=True)
elif mf == "coreH":
runner_function = lambda taskcount: Hadoop2(min(taskcount, 100000),
mapmemory=8 * 1024,
reducememory=8 * 1024,
min_alloc=2048,
xmx=3072,
mkl_num_threads=1,
queue="shared",
skipdatacheck=True,
skipsourcecheck=True)
else:
raise Exception("don't find mf=" + mf)
return runner_function