コード例 #1
0
def test_auto_backend():
    """Check the backend detection logic."""
    original_env_backend = os.environ.get('CLUSTERLIB_BACKEND', None)
    if original_env_backend is not None:
        del os.environ['CLUSTERLIB_BACKEND']
    try:
        # Check detection when no environment variable is set.
        if _which('scontrol'):
            # SLURM should be detected
            assert_equal(_get_backend('auto'), 'slurm')
        elif _which('qmod'):
            # SGE should be detected
            assert_equal(_get_backend('auto'), 'sge')
        else:
            # No backend can be detected
            assert_raises(RuntimeError, _get_backend, 'auto')

        # Check the use of the environment variable
        os.environ['CLUSTERLIB_BACKEND'] = 'slurm'
        assert_equal(_get_backend('auto'), 'slurm')

        os.environ['CLUSTERLIB_BACKEND'] = 'sge'
        assert_equal(_get_backend('auto'), 'sge')
    finally:
        # Restore the previous environment
        if original_env_backend is None:
            del os.environ['CLUSTERLIB_BACKEND']
        else:
            os.environ['CLUSTERLIB_BACKEND'] = original_env_backend
コード例 #2
0
def test_auto_backend():
    """Check the backend detection logic."""
    original_env_backend = os.environ.get('CLUSTERLIB_BACKEND', None)
    if original_env_backend is not None:
        del os.environ['CLUSTERLIB_BACKEND']
    try:
        # Check detection when no environment variable is set.
        if _which('scontrol'):
            # SLURM should be detected
            assert_equal(_get_backend('auto'), 'slurm')
        elif _which('qmod'):
            # SGE should be detected
            assert_equal(_get_backend('auto'), 'sge')
        else:
            # No backend can be detected
            assert_raises(RuntimeError, _get_backend, 'auto')

        # Check the use of the environment variable
        os.environ['CLUSTERLIB_BACKEND'] = 'slurm'
        assert_equal(_get_backend('auto'), 'slurm')

        os.environ['CLUSTERLIB_BACKEND'] = 'sge'
        assert_equal(_get_backend('auto'), 'sge')
    finally:
        # Restore the previous environment
        if original_env_backend is None:
            del os.environ['CLUSTERLIB_BACKEND']
        else:
            os.environ['CLUSTERLIB_BACKEND'] = original_env_backend
コード例 #3
0
def check_job_name_queued_or_running(job_name):
    with TemporaryDirectory() as temp_folder:
        user = getuser()

        # Launch job
        command = submit(job_command="sleep 600",
                         job_name=job_name,
                         time="700",
                         memory=500,
                         log_directory=temp_folder)
        job_id = _check_job_id(command)

        # Assert that the job has been launched
        try:
            running_jobs = queued_or_running_jobs(user=user)
            assert_in(job_name, running_jobs)
        finally:
            # Make sure to clean up even if there is a failure
            if _get_backend() == "slurm":
                subprocess.call(["scancel", job_id])

            elif _get_backend() == "sge":
                subprocess.call(["qdel", job_id])
            else:
                raise NotImplementedError("backend not implemented")
コード例 #4
0
def test_fixed_backend():
    """Check that it is possible to fix explicit backends (when valid)."""
    # Supported backends
    assert_equal(_get_backend('slurm'), 'slurm')
    assert_equal(_get_backend('sge'), 'sge')

    # Unsupported backend
    assert_raises(ValueError, _get_backend, 'hadoop')
コード例 #5
0
def test_fixed_backend():
    """Check that it is possible to fix explicit backends (when valid)."""
    # Supported backends
    assert_equal(_get_backend('slurm'), 'slurm')
    assert_equal(_get_backend('sge'), 'sge')

    # Unsupported backend
    assert_raises(ValueError, _get_backend, 'hadoop')
コード例 #6
0
def check_job_name_queued_or_running(job_name):
    with TemporaryDirectory() as temp_folder:
        user = getuser()

        # Launch job
        command = submit(job_command="sleep 600", job_name=job_name,
                         time="700", memory=500, log_directory=temp_folder)
        job_id = _check_job_id(command)

        # Assert that the job has been launched
        try:
            running_jobs = queued_or_running_jobs(user=user)
            assert_in(job_name, running_jobs)
        finally:
            # Make sure to clean up even if there is a failure
            if _get_backend() == "slurm":
                subprocess.call(["scancel", job_id])

            elif _get_backend() == "sge":
                subprocess.call(["qdel", job_id])
            else:
                raise NotImplementedError("backend not implemented")
コード例 #7
0
def test_log_output(n_trials=30):
    """Test that log output is uniform accross scheduler."""

    with TemporaryDirectory() as temp_folder:
        user = getuser()
        job_completed = False
        # Launch a sleepy SGE job
        job_name = 'ok_job'
        command = submit(job_command="echo ok",
                         job_name=job_name,
                         time="700",
                         memory=500,
                         log_directory=temp_folder)
        job_id = _check_job_id(command)

        try:
            for _ in range(n_trials):
                if job_name not in queued_or_running_jobs(user=user):
                    # job has completed, let's check the output
                    job_completed = True
                    filename = "%s.%s.txt" % (job_name, job_id)
                    assert_equal(os.listdir(temp_folder), [filename])
                    with open(op.join(temp_folder, filename)) as fhandle:
                        assert_equal(fhandle.read().strip(), "ok")
                    break
                else:
                    # Let's wait a bit before retrying
                    sleep(5)

        finally:
            # Make sure to clean up even if there is a failure
            if not job_completed:
                if _get_backend('auto') == 'slurm':
                    subprocess.call(["scancel", job_id])
                else:
                    subprocess.call(["qdel", job_id])
                raise AssertionError(
                    "job %s (%s) has not completed after 5min." %
                    (job_id, job_name))
コード例 #8
0
def test_log_output(n_trials=30):
    """Test that log output is uniform accross scheduler."""

    with TemporaryDirectory() as temp_folder:
        user = getuser()
        job_completed = False
        # Launch a sleepy SGE job
        job_name = 'ok_job'
        command = submit(job_command="echo ok", job_name=job_name,
                         time="700", memory=500,
                         log_directory=temp_folder)
        job_id = _check_job_id(command)

        try:
            for _ in range(n_trials):
                if job_name not in queued_or_running_jobs(user=user):
                    # job has completed, let's check the output
                    job_completed = True
                    filename = "%s.%s.txt" % (job_name, job_id)
                    assert_equal(os.listdir(temp_folder), [filename])
                    with open(op.join(temp_folder, filename)) as fhandle:
                        assert_equal(fhandle.read().strip(), "ok")
                    break
                else:
                    # Let's wait a bit before retrying
                    sleep(5)

        finally:
            # Make sure to clean up even if there is a failure
            if not job_completed:
                if _get_backend('auto') == 'slurm':
                    subprocess.call(["scancel", job_id])
                else:
                    subprocess.call(["qdel", job_id])
                raise AssertionError(
                    "job %s (%s) has not completed after 5min."
                    % (job_id, job_name))