예제 #1
0
def test_submit():
    """Test submit formatting function."""
    assert_equal(
        submit(job_command="python main.py", backend="sge"),
        'echo \'#!/bin/bash\npython main.py\' | qsub -N "job" -l '
        'h_rt=24:00:00 -l h_vmem=4000M')

    assert_equal(
        submit(job_command="python main.py", backend="slurm"),
        "echo '#!/bin/bash\npython main.py' | sbatch --job-name=job "
        "--time=24:00:00 --mem=4000")

    assert_equal(
        submit(job_command="python main.py", email="*****@*****.**",
               email_options="beas", backend="sge"),
        'echo \'#!/bin/bash\npython main.py\' | qsub -N "job" '
        '-l h_rt=24:00:00 -l h_vmem=4000M -M [email protected] -m beas')

    assert_equal(
        submit(job_command="python main.py", log_directory="/path/test",
               backend="sge"),
        'echo \'#!/bin/bash\npython main.py\' | qsub -N "job" '
        '-l h_rt=24:00:00 -l h_vmem=4000M -j y '
        '-o \'/path/test/$JOB_NAME.$JOB_ID.txt\'')

    assert_equal(
        submit(job_command="python main.py", log_directory="/path/test",
               backend="slurm"),
        "echo \'#!/bin/bash\npython main.py\' | sbatch --job-name=job "
        "--time=24:00:00 --mem=4000 -o /path/test/job.%j.txt")

    assert_raises(ValueError, submit, job_command="", backend="unknown")
예제 #2
0
def check_job_name_queued_or_running(job_name):
    with TemporaryDirectory() as temp_folder:
        user = getuser()

        # Launch job
        command = submit(job_command="sleep 600",
                         job_name=job_name,
                         time="700",
                         memory=500,
                         log_directory=temp_folder)
        job_id = _check_job_id(command)

        # Assert that the job has been launched
        try:
            running_jobs = queued_or_running_jobs(user=user)
            assert_in(job_name, running_jobs)
        finally:
            # Make sure to clean up even if there is a failure
            if _get_backend() == "slurm":
                subprocess.call(["scancel", job_id])

            elif _get_backend() == "sge":
                subprocess.call(["qdel", job_id])
            else:
                raise NotImplementedError("backend not implemented")
예제 #3
0
    def make_scripts(self, cmds, job_names=None):
        sys.stdout.write("Writing Scripts...\n")
        scheduled_jobs = set(queued_or_running_jobs(user=self.user))
        scripts = []

        if job_names is None:
            job_names = []

            for cmd in cmds:
                job_names.append(None)

        for index, cmd in enumerate(cmds):
            job_name = job_names[index] or self.__get_job_name(cmd)
            if job_name not in scheduled_jobs:
                script = submit(cmd,
                                memory="%s" % self.memory,
                                job_name=job_name,
                                log_directory=self.slurm_logs,
                                backend='slurm',
                                time=0,
                                shell_script="#!/usr/bin/env bash")
                script = script + " --partition=%s" % self.cluster_partition
                script = script + " --cpus-per-task=%s" % self.threads
                scripts.append(script)

                if self.verbose:
                    sys.stdout.write(script + "\n")

        return scripts
예제 #4
0
def dispatch_to_slurm(commands):
    scripts = {}

    for job_name, command in commands.iteritems():
        script = submit(command, job_name=job_name, time="0",
                        memory="{}G".format(maxmem), backend="slurm",
                        shell_script="#!/usr/bin/env bash")
        script += " --partition={}".format(partition)
        script += " --ntasks=1"
        script += " --cpus-per-task={}".format(maxcpu)
        script += " --mail-type=END,FAIL"
        script += " --mail-user={}".format(email)
        scripts[job_name] = script

    scheduled_jobs = set(queued_or_running_jobs())

    for job_name, script in scripts.iteritems():
        if job_name not in scheduled_jobs:
            if verbose:
                print("{}".format(script), file=sys.stdout)

            if not dry_run:
                subprocess.call(script, shell=True)
        else:
            print("{} already running, skipping".format(job_name),
                  file=sys.stderr)
예제 #5
0
def dispatchscript(cmd):
    script = submit(cmd, memory="240000", job_name="bwa_mem", backend="slurm",
                    time="0", shell_script="#!/usr/bin/env bash",
                    log_directory=slurm_logs)
    script = script + " --partition=bigmemm"
    script = script + " --cpus-per-task=30"
    print "\nCalling:\n%s\n" % script
    call("%s" % script, shell=True)
예제 #6
0
파일: submit.py 프로젝트: noa/brute
def get_submission_script(cmd, name, workdir, config):
    backend = config.get("brute", "env")
    assert cmd
    assert name
    script = None
    if backend == "sge":
        script = submit(cmd,
                        job_name=name,
                        time=config.get("sge","time"),
                        memory=config.getint("sge","memory"),
                        backend=backend,
                        log_directory=workdir)
    elif backend == "slurm":
        script = submit(cmd,
                        job_name=name,
                        time=config.get("slurm","time"),
                        memory=config.getint("slurm","memory"),
                        backend=backend,
                        log_directory=workdir)

        if config.has_option("slurm","gres"):
            script += " --gres=" + config.get("slurm", "gres")

        if config.has_option("slurm","partition"):
            script += " --partition=" + config.get("slurm", "partition")

        if config.has_option("slurm", "account"):
            script += " --account=" + config.get("slurm", "account")

        script += " --cpus-per-task=" + str(config.getint("slurm", "cpus-per-task"))
        script += " --ntasks-per-node=" + str(config.getint("slurm", "ntasks-per-node"))

    elif backend == "local":
        script = """#! /usr/bin/env bash
                    cd $(dirname $0)
                    CMD="%s"
                    $CMD &> %s""" % (cmd, os.path.join(workdir, "log.txt"))
    else:
        print("[fatal] unknown backend: " + str(backend))
        sys.exit(1)

    return script
예제 #7
0
def test_submit():
    """Test submit formatting function."""
    assert_equal(
        submit(job_command="python main.py", backend="sge"),
        'echo \'#!/bin/bash\npython main.py\' | qsub -N "job" -l '
        'h_rt=24:00:00 -l h_vmem=4000M')

    assert_equal(
        submit(job_command="python main.py", backend="slurm"),
        "echo '#!/bin/bash\npython main.py' | sbatch --job-name=job "
        "--time=24:00:00 --mem=4000")

    assert_equal(
        submit(job_command="python main.py",
               email="*****@*****.**",
               email_options="beas",
               backend="sge"),
        'echo \'#!/bin/bash\npython main.py\' | qsub -N "job" '
        '-l h_rt=24:00:00 -l h_vmem=4000M -M [email protected] -m beas')

    assert_equal(
        submit(job_command="python main.py",
               log_directory="/path/test",
               backend="sge"),
        'echo \'#!/bin/bash\npython main.py\' | qsub -N "job" '
        '-l h_rt=24:00:00 -l h_vmem=4000M -j y '
        '-o \'/path/test/$JOB_NAME.$JOB_ID.txt\'')

    assert_equal(
        submit(job_command="python main.py",
               log_directory="/path/test",
               backend="slurm"),
        "echo \'#!/bin/bash\npython main.py\' | sbatch --job-name=job "
        "--time=24:00:00 --mem=4000 -o /path/test/job.%j.txt")

    assert_raises(ValueError, submit, job_command="", backend="unknown")
예제 #8
0
def dispatch(commands):
    scripts = {}

    for job_name, command in commands.iteritems():
        script = submit(command, job_name=job_name,
                        time="0", memory=max_memory + "G", backend="slurm",
                        shell_script="#!/usr/bin/env bash")
        script = script + " --partition=bigmemm"
        scripts[job_name] = script

    scheduled_jobs = set(queued_or_running_jobs())

    for job_name, script in scripts.iteritems():
        if job_name not in scheduled_jobs:
            sys.stdout.write("\n{}\n".format(script))
            subprocess.call(script, shell=True)
        else:
            sys.stderr.write("{} running, skipping.\n".format(job_name))
예제 #9
0
def write_scripts(commands):
    scripts = {}

    for job_name, command in commands.iteritems():
        script = submit(command,
                        job_name=job_name,
                        time="0",
                        memory=max_memory,
                        backend="slurm",
                        shell_script="#!/usr/bin/env bash")
        script += " --partition={}".format(partition)
        script += " --ntasks=1"
        script += " --cpus-per-task={}".format(max_cpus)
        script += " --mail-type=END,FAIL"
        script += " --mail-user={}".format(email)
        scripts[job_name] = script

    return scripts
예제 #10
0
def dispatch(filelist, commands):
    scripts = []

    for i, command in enumerate(commands):
        script = submit(command, job_name="samtools_sort_%s" % filelist[i],
                        time="0", memory=240000, backend="slurm",
                        shell_script="#!/usr/bin/env bash")
        script = script + " --cpus-per-task=12"
        script = script + " --ntasks=1"
        script = script + " --partition=bigmemm"
        scripts.append(script)

    scheduled_jobs = set(queued_or_running_jobs())

    for i, script in enumerate(scripts):
        if "samtools_sort_%s" % filelist[i] not in scheduled_jobs:
            sys.stdout.write("\n%s\n" % script)
            subprocess.call(script, shell=True)
        else:
            sys.stderr.write("Job name 'samtools_sort_%s' found in queued or \
                             running jobs list" % filelist[i])
예제 #11
0
def check_job_name_queued_or_running(job_name):
    with TemporaryDirectory() as temp_folder:
        user = getuser()

        # Launch job
        command = submit(job_command="sleep 600", job_name=job_name,
                         time="700", memory=500, log_directory=temp_folder)
        job_id = _check_job_id(command)

        # Assert that the job has been launched
        try:
            running_jobs = queued_or_running_jobs(user=user)
            assert_in(job_name, running_jobs)
        finally:
            # Make sure to clean up even if there is a failure
            if _get_backend() == "slurm":
                subprocess.call(["scancel", job_id])

            elif _get_backend() == "sge":
                subprocess.call(["qdel", job_id])
            else:
                raise NotImplementedError("backend not implemented")
예제 #12
0
def test_log_output(n_trials=30):
    """Test that log output is uniform accross scheduler."""

    with TemporaryDirectory() as temp_folder:
        user = getuser()
        job_completed = False
        # Launch a sleepy SGE job
        job_name = 'ok_job'
        command = submit(job_command="echo ok",
                         job_name=job_name,
                         time="700",
                         memory=500,
                         log_directory=temp_folder)
        job_id = _check_job_id(command)

        try:
            for _ in range(n_trials):
                if job_name not in queued_or_running_jobs(user=user):
                    # job has completed, let's check the output
                    job_completed = True
                    filename = "%s.%s.txt" % (job_name, job_id)
                    assert_equal(os.listdir(temp_folder), [filename])
                    with open(op.join(temp_folder, filename)) as fhandle:
                        assert_equal(fhandle.read().strip(), "ok")
                    break
                else:
                    # Let's wait a bit before retrying
                    sleep(5)

        finally:
            # Make sure to clean up even if there is a failure
            if not job_completed:
                if _get_backend('auto') == 'slurm':
                    subprocess.call(["scancel", job_id])
                else:
                    subprocess.call(["qdel", job_id])
                raise AssertionError(
                    "job %s (%s) has not completed after 5min." %
                    (job_id, job_name))
예제 #13
0
def test_log_output(n_trials=30):
    """Test that log output is uniform accross scheduler."""

    with TemporaryDirectory() as temp_folder:
        user = getuser()
        job_completed = False
        # Launch a sleepy SGE job
        job_name = 'ok_job'
        command = submit(job_command="echo ok", job_name=job_name,
                         time="700", memory=500,
                         log_directory=temp_folder)
        job_id = _check_job_id(command)

        try:
            for _ in range(n_trials):
                if job_name not in queued_or_running_jobs(user=user):
                    # job has completed, let's check the output
                    job_completed = True
                    filename = "%s.%s.txt" % (job_name, job_id)
                    assert_equal(os.listdir(temp_folder), [filename])
                    with open(op.join(temp_folder, filename)) as fhandle:
                        assert_equal(fhandle.read().strip(), "ok")
                    break
                else:
                    # Let's wait a bit before retrying
                    sleep(5)

        finally:
            # Make sure to clean up even if there is a failure
            if not job_completed:
                if _get_backend('auto') == 'slurm':
                    subprocess.call(["scancel", job_id])
                else:
                    subprocess.call(["qdel", job_id])
                raise AssertionError(
                    "job %s (%s) has not completed after 5min."
                    % (job_id, job_name))
예제 #14
0
# clusterlib_launcher.py

import sys
from clusterlib.scheduler import queued_or_running_jobs
from clusterlib.scheduler import submit
from clusterlib.storage import sqlite3_loads
from clusterlib_main import NOSQL_PATH

if __name__ == "__main__":
    scheduled_jobs = set(queued_or_running_jobs())
    done_jobs = sqlite3_loads(NOSQL_PATH)

    for param in range(100):
        job_name = "job-param=%s" % param
        job_command = "%s clusterlib_main.py --param %s" % (sys.executable,
                                                            param)

        if job_name not in scheduled_jobs and job_command not in done_jobs:
            script = submit(job_command, job_name=job_name)
            print(script)

            # Uncomment those lines to launch the jobs
            # import os
            # os.system(script)
예제 #15
0
# clusterlib_launcher.py

import sys
from clusterlib.scheduler import queued_or_running_jobs
from clusterlib.scheduler import submit
from clusterlib.storage import sqlite3_loads
from clusterlib_main import NOSQL_PATH

if __name__ == "__main__":
    scheduled_jobs = set(queued_or_running_jobs())
    done_jobs = sqlite3_loads(NOSQL_PATH)

    for param in range(100):
        job_name = "job-param=%s" % param
        job_command = "%s clusterlib.py --param %s" % (sys.executable,
                                                       param)

        if job_name not in scheduled_jobs and job_command not in done_jobs:
            script = submit(job_command, job_name=job_name)
            print(script)

            # Uncomment those lines to launch the jobs
            # import os
            # os.system(script)
예제 #16
0
        else:
            n_jobs_launched += 1
            cmd_parameters = " ".join("--%s %s" % (key, parameters[key])
                                      for key in sorted(parameters))

            scripts_args = parse_arguments(shlex.split(cmd_parameters))
            if make_hash(scripts_args) != job_hash:
                pprint(scripts_args)
                pprint(parameters)
                raise ValueError("hash are not equal, all parameters are "
                                 "not specified.")

            cmd = submit(job_command="%s main.py %s" %
                         (sys.executable, cmd_parameters),
                         job_name=job_hash,
                         time="100:00:00",
                         memory=24000,
                         log_directory=LOG_DIRECTORY,
                         backend="slurm")

            if not args["debug"]:
                os.system(cmd)
            elif args["verbose"]:
                print("[launched] %s" % job_hash)
                print(cmd)

                if os.path.exists(
                        os.path.join(LOG_DIRECTORY, "%s.txt" % job_hash)):
                    os.system("cat %s" %
                              os.path.join(LOG_DIRECTORY, "%s.txt" % job_hash))
예제 #17
0
        else:
            n_jobs_launched += 1
            cmd_parameters = " ".join("--%s %s" % (key, parameters[key])
                                      for key in sorted(parameters))

            scripts_args = parse_arguments(shlex.split(cmd_parameters))
            if make_hash(scripts_args) != job_hash:
                pprint(scripts_args)
                pprint(parameters)
                raise ValueError("hash are not equal, all parameters are "
                                 "not specified.")

            cmd = submit(job_command="%s main.py %s"
                                     % (sys.executable, cmd_parameters),
                         job_name=job_hash,
                         time="100:00:00",
                         memory=24000,
                         log_directory=LOG_DIRECTORY,
                         backend="slurm")

            if not args["debug"]:
                os.system(cmd)
            elif args["verbose"]:
                print("[launched] %s" % job_hash)
                print(cmd)

                if os.path.exists(os.path.join(LOG_DIRECTORY,
                                               "%s.txt" % job_hash)):
                    os.system("cat %s" % os.path.join(LOG_DIRECTORY,
                                                      "%s.txt" % job_hash))
예제 #18
0
    for job_hash, parameters in list(to_launch.items())[:max_n_launch]:
        cmd_parameters = " ".join("--%s %s" % (key, parameters[key])
                                  for key in sorted(parameters))

        scripts_args = parse_arguments(shlex.split(cmd_parameters))
        if make_hash(scripts_args) != job_hash:
            pprint(scripts_args)
            pprint(parameters)
            raise ValueError("hash are not equal, all parameters are "
                             "not specified.")

        cmd = submit(job_command=" ".join(
            [sys.executable,
             os.path.abspath("main.py"), cmd_parameters]),
                     job_name=job_hash,
                     time="%s:00:00" % time[job_hash],
                     memory=memory[job_hash],
                     log_directory=LOG_DIRECTORY,
                     backend="slurm")

        cmd += select_queue(memory[job_hash], time[job_hash])

        if not args["debug"]:
            os.system(cmd)
            n_jobs_launched += 1

        elif args["verbose"]:
            print("[launched] %s " % (job_hash, ))
            print(cmd)

    print("\nSummary launched")