def test_custom_lsf_per_core_env(tmpdir):
jobstore = tmpdir.join("jobstore").strpath
log = tmpdir.join("log.txt").strpath
options = parsers.ToilBaseArgumentParser().parse_args(
[jobstore, "--logFile", log])
options.batchSystem = "custom_lsf"
options.disableCaching = True
# Set total memory per job
os.environ["TOIL_CONTAINER_LSF_PER_CORE"] = "N"
job_1 = jobs.ContainerJob(options, runtime=1, cores=2, memory="10G")
jobs.ContainerJob.Runner.startToil(job_1, options)
with open(log, "rt", encoding="utf-8") as f:
assert "-M 10000MB -n 2" in f.read()
# Set total memory per core
os.environ["TOIL_CONTAINER_LSF_PER_CORE"] = "Y"
job_2 = jobs.ContainerJob(options, runtime=1, cores=2, memory="10G")
jobs.ContainerJob.Runner.startToil(job_2, options)
with open(log, "rt", encoding="utf-8") as f:
assert "-M 5000MB -n 2" in f.read()
# Use per_core_reservation() from lsf config
del os.environ["TOIL_CONTAINER_LSF_PER_CORE"]
job_3 = jobs.ContainerJob(options, runtime=1, cores=2, memory="10G")
jobs.ContainerJob.Runner.startToil(job_3, options)
with open(log, "rt", encoding="utf-8") as f:
assert ("-M 5000MB -n 2"
if per_core_reservation() else "-M 10000MB -n 2") in f.read()
def assert_image_call(image_attribute, image, tmpdir):
"""Get options namespace."""
options = argparse.Namespace()
options.workDir = tmpdir.mkdir("working_dir").strpath
setattr(options, image_attribute, image)
# create job and options
vol1 = tmpdir.mkdir("vol1")
vol2 = tmpdir.mkdir("vol2")
options.volumes = [
(vol1.strpath, "/vol1"),
(vol2.strpath, "/vol2"),
]
vol1.join("foo").write("bar")
job = jobs.ContainerJob(options)
# test cwd
assert "bin" in job.call(["ls", ".."], cwd="/bin", check_output=True)
# test volume
assert "bar" in job.call(["cat", "/vol1/foo"], check_output=True)
assert job.call(["ls"]) == 0
# test env
cmd = ["bash", "-c", "echo $FOO"]
assert "BAR" in job.call(cmd, env={"FOO": "BAR"}, check_output=True)
# check subprocess.CalledProcessError
with pytest.raises(exceptions.SystemCallError):
job.call(["rm", "/bin"])
# check OSError
with pytest.raises(exceptions.SystemCallError):
job.call(["florentino-ariza"])
# test both singularity and docker raiser error
options = argparse.Namespace()
options.docker = "foo"
options.singularity = "bar"
job = jobs.ContainerJob(options)
with pytest.raises(exceptions.UsageError) as error:
job.call(["foo", "bar"])
assert "Both docker and singularity can't be set" in str(error.value)
def test_custom_lsf_batch_system(tmpdir):
jobstore = tmpdir.join("jobstore").strpath
log = tmpdir.join("log.txt").strpath
options = parsers.ToilBaseArgumentParser().parse_args([jobstore, "--logFile", log])
options.batchSystem = "CustomLSF"
job = jobs.ContainerJob(options, memory="10G", runtime=1)
jobs.ContainerJob.Runner.startToil(job, options)
with open(log) as f:
assert "-W 1" in f.read()
def test_custom_lsf_batch_system(tmpdir):
jobstore = tmpdir.join("jobstore").strpath
log = tmpdir.join("log.txt").strpath
options = parsers.ToilBaseArgumentParser().parse_args(
[jobstore, "--logFile", log])
options.batchSystem = "custom_lsf"
options.disableCaching = True
job = jobs.ContainerJob(options, runtime=1)
jobs.ContainerJob.Runner.startToil(job, options)
with open(log, encoding="utf-8") as f:
assert "-W 1" in f.read()
def test_call_uses_subprocess():
options = argparse.Namespace()
job = jobs.ContainerJob(options)
assert job.call(["ls"]) == 0
assert "bin" in job.call(["ls", "/"], check_output=True)
# check subprocess.CalledProcessError
with pytest.raises(exceptions.SystemCallError):
job.call(["rm", "/bin"])
# check OSError
with pytest.raises(exceptions.SystemCallError):
job.call(["florentino-ariza"])
def test_resources_are_encoded():
options = argparse.Namespace()
options.batchSystem = "CustomLSF"
job = jobs.ContainerJob(options, runtime=1, unitName="foo")
assert lsf._RESOURCES_START_TAG in job.unitName
assert lsf._RESOURCES_CLOSE_TAG in job.unitName
def test_displayname_set_to_class_name_by_default():
options = argparse.Namespace()
job = jobs.ContainerJob(options)
assert job.displayName == job.__class__.__name__
def test_jobname_set_to_class_name_by_default():
options = argparse.Namespace()
job = jobs.ContainerJob(options)
assert job.jobName == "containerjob"