def from_dict(cls, d):
qa = QueueAdapter.from_dict(d['queue_adapter']) if d['queue_adapter'] is not None else None
return cls(job_rundir=d['job_rundir'], qout_file=d['qout_file'], qerr_file=d['qerr_file'], queue_adapter=qa,
max_mem_per_proc_mb=d['max_mem_per_proc_mb'],
mem_per_proc_increase_mb=d['mem_per_proc_increase_mb'],
max_master_mem_overhead_mb=d['max_master_mem_overhead_mb'],
master_mem_overhead_increase_mb=d['master_mem_overhead_increase_mb'])
def test_walltime_handler(self):
# Simple test from a queue_adapter
f = open(os.path.join(test_dir, "fw_handlers", "walltime", "PBS_qadapter.json"))
dd = json.load(f)
f.close()
qad = QueueAdapter.from_dict(dd)
walltime_handler = WalltimeHandler(job_rundir=os.path.join(test_dir, "fw_handlers", "walltime"),
qout_file='queue.qout', qerr_file='queue.qerr', queue_adapter=qad,
max_timelimit=1200, timelimit_increase=600)
self.assertTrue(walltime_handler.check())
# Test with a given Firework
walltime_handler = WalltimeHandler(job_rundir='.',
qout_file='queue.qout', qerr_file='queue.qerr', queue_adapter=None,
max_timelimit=1200, timelimit_increase=600)
# Get the Firework from json file
f = open(os.path.join(test_dir, 'fw_handlers', 'walltime', 'sleep_fw.json'))
dd = json.load(f)
f.close()
fw_to_check = Firework.from_dict(dd)
# Hack the launch_dir so that it points to the directory where the queue.qout is
fw_to_check.launches[-1].launch_dir = os.path.join(test_dir, fw_to_check.launches[-1].launch_dir)
walltime_handler.src_setup(fw_spec={}, fw_to_check=fw_to_check)
self.assertTrue(walltime_handler.check())
actions = walltime_handler.correct()
self.assertEqual(actions, {'errors': ['WalltimeHandler'],
'actions': [{'action': {'_set': {'timelimit': 660.0}},
'object': {'source': 'fw_spec', 'key': 'qtk_queueadapter'},
'action_type': 'modify_object'}]})
def from_dict(cls, d):
qa = QueueAdapter.from_dict(
d['queue_adapter']) if d['queue_adapter'] is not None else None
return cls(job_rundir=d['job_rundir'],
qout_file=d['qout_file'],
qerr_file=d['qerr_file'],
queue_adapter=qa,
max_timelimit=d['max_timelimit'],
timelimit_increase=d['timelimit_increase'])
def test_manager(self):
"""Testing abirun.py manager"""
env = self.get_env()
no_logo_colors = ["--no-logo", "--no-colors"]
# Test doc_manager
env.run(self.script, ".", "doc_manager", self.loglevel, self.verbose, *no_logo_colors)
for qtype in QueueAdapter.all_qtypes():
env.run(self.script, ".", "doc_manager", qtype, self.loglevel, self.verbose, *no_logo_colors)
# Test doc_sheduler
env.run(self.script, ".", "doc_scheduler", self.loglevel, self.verbose, *no_logo_colors)
def test_manager(self):
"""Testing abirun.py manager"""
env = self.get_env()
no_logo_colors = ["--no-logo", "--no-colors"]
# Test doc_manager
env.run(self.script, ".", "doc_manager", self.loglevel, self.verbose,
*no_logo_colors)
for qtype in QueueAdapter.all_qtypes():
env.run(self.script, ".", "doc_manager", qtype, self.loglevel,
self.verbose, *no_logo_colors)
# Test doc_sheduler
env.run(self.script, ".", "doc_scheduler", self.loglevel, self.verbose,
*no_logo_colors)
def test_without_flow(self):
"""Testing abirun.py commands without flow"""
env = self.get_env()
no_logo_colors = ["--no-logo", "--no-colors"]
# Test doc_manager
r = env.run(self.script, "doc_manager", self.loglevel, self.verbose, *no_logo_colors,
expect_stderr=self.expect_stderr)
for qtype in QueueAdapter.all_qtypes():
r = env.run(self.script, ".", "doc_manager", qtype, self.loglevel, self.verbose, *no_logo_colors,
expect_stderr=self.expect_stderr)
# Test doc_sheduler
r = env.run(self.script, "doc_scheduler", self.loglevel, self.verbose, *no_logo_colors,
expect_stderr=self.expect_stderr)
# Test abibuild
r = env.run(self.script, "abibuild", self.loglevel, self.verbose, *no_logo_colors,
expect_stderr=self.expect_stderr)
def test_without_flow(self):
"""Testing abirun.py commands without flow"""
env = self.get_env()
no_logo_colors = ["--no-logo", "--no-colors"]
# Test doc_manager
r = env.run(self.script, "doc_manager", self.loglevel, self.verbose, *no_logo_colors,
expect_stderr=self.expect_stderr)
for qtype in QueueAdapter.all_qtypes():
r= env.run(self.script, ".", "doc_manager", qtype, self.loglevel, self.verbose, *no_logo_colors,
expect_stderr=self.expect_stderr)
# Test doc_sheduler
r = env.run(self.script, "doc_scheduler", self.loglevel, self.verbose, *no_logo_colors,
expect_stderr=self.expect_stderr)
# Test abibuild
r = env.run(self.script, "abibuild", self.loglevel, self.verbose, *no_logo_colors,
expect_stderr=self.expect_stderr)
def test_base(self):
"""unit tests for Qadapter subclasses. A more complete coverage would require integration testing."""
self.maxDiff = None
aequal, atrue, afalse = self.assertEqual, self.assertTrue, self.assertFalse
sub_classes = QueueAdapter.__subclasses__()
# Test if we can instantiate the concrete classes with the abc protocol.
for subc in sub_classes:
print("subclass: ", subc)
# Create the adapter subclass.
self.QDICT["queue"]["qtype"] = subc.QTYPE
qad = make_qadapter(**self.QDICT)
print(qad)
hw = qad.hw
giga = 1024
# Test the programmatic interface used to change job parameters.
aequal(qad.num_launches, 0)
afalse(qad.has_omp)
atrue(qad.has_mpi)
qad.set_mpi_procs(2)
aequal(qad.mpi_procs, 2)
atrue(qad.pure_mpi)
afalse(qad.pure_omp)
afalse(qad.hybrid_mpi_omp)
aequal(qad.mem_per_proc, giga)
qad.set_mem_per_proc(2 * giga)
aequal(qad.mem_per_proc, 2 * giga)
aequal(qad.timelimit, 120)
# Enable OMP
qad.set_omp_threads(2)
aequal(qad.omp_threads, 2)
atrue(qad.has_omp)
afalse(qad.pure_mpi)
afalse(qad.pure_omp)
atrue(qad.hybrid_mpi_omp)
atrue(
qad.hw.can_use_omp_threads(hw.sockets_per_node *
hw.cores_per_socket))
afalse(
qad.hw.can_use_omp_threads(hw.sockets_per_node *
hw.cores_per_socket + 1))
# Test the creation of the script
script = qad.get_script_str("job.sh",
"/launch/dir",
"executable",
"qout_path",
"qerr_path",
stdin="STDIN",
stdout="STDOUT",
stderr="STDERR")
# Test whether qad can be serialized with Pickle.
deserialized_qads = self.serialize_with_pickle(qad, test_eq=False)
for new_qad in deserialized_qads:
new_script = new_qad.get_script_str("job.sh",
"/launch/dir",
"executable",
"qout_path",
"qerr_path",
stdin="STDIN",
stdout="STDOUT",
stderr="STDERR")
aequal(new_script, script)
# Test can_run and distribute
# The hardware has num_nodes=3, sockets_per_node=2, cores_per_socket=4, mem_per_node="8 Gb"
afalse(
qad.can_run_pconf(
ParalConf(mpi_ncpus=hw.num_cores + 1,
omp_ncpus=1,
mem_per_cpu=0.1)))
afalse(
qad.can_run_pconf(
ParalConf(mpi_ncpus=4, omp_ncpus=9, mem_per_cpu=0.1)))
afalse(
qad.can_run_pconf(
ParalConf(mpi_ncpus=4, omp_ncpus=1,
mem_per_cpu=10 * giga)))
d = qad.distribute(mpi_procs=4, omp_threads=1, mem_per_proc=giga)
assert d.num_nodes == 1 and d.mpi_per_node == 4 and d.exact
d = qad.distribute(mpi_procs=16, omp_threads=1, mem_per_proc=giga)
assert d.num_nodes == 2 and d.mpi_per_node == 8 and d.exact
# not enough memory per node but can distribute.
d = qad.distribute(mpi_procs=8,
omp_threads=1,
mem_per_proc=2 * giga)
assert d.num_nodes == 2 and d.mpi_per_node == 4 and not d.exact
# mem_per_proc > mem_per_node!
with self.assertRaises(qad.Error):
d = qad.distribute(mpi_procs=9,
omp_threads=1,
mem_per_proc=10 * giga)
# TODO
# not commensurate with node
#d = qad.distribute(mpi_procs=9, omp_threads=1, mem_per_proc=giga)
#assert d.num_nodes == 3 and d.mpi_per_node == 3 and not d.exact
with self.assertRaises(qad.Error):
qad.set_mpi_procs(25)
qad.validate()
with self.assertRaises(qad.Error):
qad.set_mpi_procs(100)
qad.validate()
with self.assertRaises(qad.Error):
qad.set_omp_threads(10)
qad.validate()
with self.assertRaises(qad.Error):
qad.set_mem_per_proc(9 * giga)
qad.validate()
# Test if one can register a customized class.
class MyAdapter(SlurmAdapter):
QTYPE = "myslurm"
SlurmAdapter.register(MyAdapter)
assert issubclass(MyAdapter, QueueAdapter)
self.QDICT["queue"]["qtype"] = "myslurm"
qad = make_qadapter(**self.QDICT)
assert isinstance(qad, MyAdapter)
def test_base(self):
"""unit tests for Qadapter subclasses. A more complete coverage would require integration testing."""
self.maxDiff = None
aequal, atrue, afalse = self.assertEqual, self.assertTrue, self.assertFalse
sub_classes = QueueAdapter.__subclasses__()
# Test if we can instantiate the concrete classes with the abc protocol.
for subc in sub_classes:
print("subclass: ", subc)
# Create the adapter subclass.
self.QDICT["queue"]["qtype"] = subc.QTYPE
qad = make_qadapter(**self.QDICT)
print(qad)
hw = qad.hw
giga = 1024
# Test the programmatic interface used to change job parameters.
aequal(qad.num_launches, 0)
afalse(qad.has_omp)
atrue(qad.has_mpi)
qad.set_mpi_procs(2)
aequal(qad.mpi_procs, 2)
atrue(qad.pure_mpi)
afalse(qad.pure_omp)
afalse(qad.hybrid_mpi_omp)
aequal(qad.mem_per_proc, giga)
qad.set_mem_per_proc(2 * giga)
aequal(qad.mem_per_proc, 2 * giga)
aequal(qad.timelimit, 120)
# Enable OMP
qad.set_omp_threads(2)
aequal(qad.omp_threads, 2)
atrue(qad.has_omp)
afalse(qad.pure_mpi)
afalse(qad.pure_omp)
atrue(qad.hybrid_mpi_omp)
atrue(qad.hw.can_use_omp_threads(hw.sockets_per_node * hw.cores_per_socket))
afalse(qad.hw.can_use_omp_threads(hw.sockets_per_node * hw.cores_per_socket + 1))
# Test the creation of the script
script = qad.get_script_str("job.sh", "/launch/dir", "executable", "qout_path", "qerr_path",
stdin="STDIN", stdout="STDOUT", stderr="STDERR")
# Test whether qad can be serialized with Pickle.
deserialized_qads = self.serialize_with_pickle(qad, test_eq=False)
for new_qad in deserialized_qads:
new_script = new_qad.get_script_str("job.sh", "/launch/dir", "executable", "qout_path", "qerr_path",
stdin="STDIN", stdout="STDOUT", stderr="STDERR")
aequal(new_script, script)
# Test can_run and distribute
# The hardware has num_nodes=3, sockets_per_node=2, cores_per_socket=4, mem_per_node="8 Gb"
afalse(qad.can_run_pconf(ParalConf(mpi_ncpus=hw.num_cores+1, omp_ncpus=1, mem_per_cpu=0.1)))
afalse(qad.can_run_pconf(ParalConf(mpi_ncpus=4, omp_ncpus=9, mem_per_cpu=0.1)))
afalse(qad.can_run_pconf(ParalConf(mpi_ncpus=4, omp_ncpus=1, mem_per_cpu=10 * giga)))
d = qad.distribute(mpi_procs=4, omp_threads=1, mem_per_proc=giga)
assert d.num_nodes == 1 and d.mpi_per_node == 4 and d.exact
d = qad.distribute(mpi_procs=16, omp_threads=1, mem_per_proc=giga)
assert d.num_nodes == 2 and d.mpi_per_node == 8 and d.exact
# not enough memory per node but can distribute.
d = qad.distribute(mpi_procs=8, omp_threads=1, mem_per_proc=2 * giga)
assert d.num_nodes == 2 and d.mpi_per_node == 4 and not d.exact
# mem_per_proc > mem_per_node!
with self.assertRaises(qad.Error):
d = qad.distribute(mpi_procs=9, omp_threads=1, mem_per_proc=10 * giga)
# TODO
# not commensurate with node
#d = qad.distribute(mpi_procs=9, omp_threads=1, mem_per_proc=giga)
#assert d.num_nodes == 3 and d.mpi_per_node == 3 and not d.exact
with self.assertRaises(qad.Error):
qad.set_mpi_procs(25)
qad.validate()
with self.assertRaises(qad.Error):
qad.set_mpi_procs(100)
qad.validate()
with self.assertRaises(qad.Error):
qad.set_omp_threads(10)
qad.validate()
with self.assertRaises(qad.Error):
qad.set_mem_per_proc(9 * giga)
qad.validate()
# Test if one can register a customized class.
class MyAdapter(SlurmAdapter):
QTYPE = "myslurm"
SlurmAdapter.register(MyAdapter)
assert issubclass(MyAdapter, QueueAdapter)
self.QDICT["queue"]["qtype"] = "myslurm"
qad = make_qadapter(**self.QDICT)
assert isinstance(qad, MyAdapter)
def from_dict(cls, d):
qa = QueueAdapter.from_dict(d['queue_adapter']) if d['queue_adapter'] is not None else None
return cls(job_rundir=d['job_rundir'], qout_file=d['qout_file'], qerr_file=d['qerr_file'], queue_adapter=qa,
max_timelimit=d['max_timelimit'],
timelimit_increase=d['timelimit_increase'])
