def is_running():
# Does it look like tools/ci/setup-slurm-container.sh was called?
try:
out, _ = Runner().run(
["docker", "port", "reproman-slurm-container"],
expect_fail=True, expect_stderr=True)
except (CommandError, FileNotFoundError):
return False
return out.strip()
def check_run_and_get_output(cmd):
runner = Runner()
try:
# suppress log output happen it was set to high values
with patch.dict('os.environ', {'REPROMAN_LOGLEVEL': 'WARN'}):
output = runner.run(["reproman", "--help"])
except CommandError as e:
raise AssertionError("'reproman --help' failed to start normally. "
"Exited with %d and output %s" %
(e.code, (e.stdout, e.stderr)))
return output
def is_running():
try:
Runner().run(["condor_status"],
expect_fail=True, expect_stderr=True)
except (CommandError, FileNotFoundError):
return False
return True
def test_create_and_start(tmpdir):
runner = Runner()
tmpdir = str(tmpdir)
cfg_file = op.join(tmpdir, "custom.cfg")
inventory_file = op.join(tmpdir, "inventory.yml")
with open(cfg_file, "w") as cfg_fh:
cfg_fh.write("[general]\ninventory_file = {}\n".format(inventory_file))
def run_reproman(args):
runner(["reproman", "--config", cfg_file] + args, expect_stderr=True)
run_reproman(["create", "--resource-type=shell", "myshell"])
with open(inventory_file) as ifh:
dumped = ifh.read()
assert "myshell" in dumped
assert "id" in dumped
# Running with a different config fails ...
empty_cfg_file = op.join(tmpdir, "empty.cfg")
with open(empty_cfg_file, "w"):
pass
with swallow_logs(new_level=logging.ERROR) as cml:
with pytest.raises(CommandError):
runner(
["reproman", "--config", empty_cfg_file, "start", "myshell"])
if os.environ.get("REPROMAN_LOGTARGET", "stderr") == "stderr":
assert "ResourceNotFoundError" in cml.out
# ... but using the same config works.
run_reproman(["start", "myshell"])
def docker_fixture():
"""The actual fixture code generated by get_docker_fixture
on setup, this fixture ensures that a docker container is running
and starts one if necessary.
Fixture yields parameters of the container with a `custom` field passed
into the `get_docker_container`.
on teardown, this fixture stops the docker container it started
"""
skip_if_no_network()
args = [
'docker',
'run',
'-d',
'--rm',
]
if seccomp_unconfined:
args.extend(['--security-opt', 'seccomp=unconfined'])
params = {}
if name:
args += ['--name', name]
params['name'] = name
if portmaps:
for from_to in portmaps.items():
args += ['-p', '%d:%d' % from_to]
params['port'] = from_to[0]
args += [image]
stdout, _ = Runner().run(args)
params['container_id'] = container_id = stdout.strip()
params['custom'] = custom_params
yield params
Runner().run(['docker', 'stop', container_id])
def revision(self):
# svn info doesn't give the current revision
# (see http://svnbook.red-bean.com/en/1.7/svn.tour.history.html)
# so we need to run svn log
if not hasattr(self, '_revision'):
runner = Runner()
log = runner(['svn', 'log', '^/', '-l', '1'], cwd=self.path)[0]
lines = log.strip().split('\n')
if len(lines) == 1:
self._revision = None
else:
revision = lines[1].split()[0]
if revision.startswith('r') and revision[1:].isdigit():
self._revision = int(revision[1:])
else:
self._revision = revision[1:]
return self._revision
def chown(self, path, uid=-1, gid=-1, recursive=False, remote=True):
"""Set the user and gid of a path
"""
uid = int(uid) # Command line parameters getting passed as type str
gid = int(gid)
if uid == -1 and gid > -1:
command = ['chgrp']
else:
command = ['chown']
if recursive: command += ["-R"]
if uid > -1 and gid > -1: command += ["{}.{}".format(uid, gid)]
elif uid > -1: command += [uid]
elif gid > -1: command += [gid]
else: raise CommandError(cmd='chown', msg="Invalid command \
parameters.")
command += [path]
if remote:
self.execute_command(command)
else:
# Run on the local file system
Runner().run(command)
def venv_test_dir():
dirs = AppDirs('reproman')
test_dir = os.path.join(dirs.user_cache_dir, 'venv_test')
if os.path.exists(test_dir):
return test_dir
runner = Runner()
runner.run(["mkdir", "-p", test_dir])
pymod_dir = os.path.join(test_dir, "minimal_pymodule")
create_pymodule(pymod_dir)
with chpwd(test_dir):
runner.run(["virtualenv", "--python", PY_VERSION, "venv0"])
runner.run(["virtualenv", "--python", PY_VERSION, "venv1"])
runner.run(["./venv0/bin/pip", "install", "pyyaml"])
runner.run(["./venv0/bin/pip", "install", "-e", pymod_dir])
runner.run(["./venv1/bin/pip", "install", "attrs"])
# Make sure we're compatible with older pips.
runner.run(["./venv1/bin/pip", "install", "pip==9.0.3"])
return test_dir
def git_repo_fixture(kind="default", scope="function"):
"""Create a Git repository fixture.
Parameters
----------
kind : {"empty", "default", "pair"}, optional
Kind git repository.
- empty: a repository with no commits.
- default: a repository with three commits, each adding one of
its three files: "foo", "bar", and "subdir/baz".
- pair: a (local, remote) pair of Git repos. The repos have
the same structure as the "default" repo, but "local" has a
remote "origin" with a URL that points to "remote".
scope : {"function", "class", "module", "session"}, optional
A `pytest.fixture` scope argument.
Returns
-------
A fixture function.
"""
runner = Runner()
def setup_user():
# Set the user in the local Git configuration rather than
# through environmental variables so that test functions using
# this fixture can make commits under the same user.
runner(["git", "config", "user.name", "A U Thor"])
runner(["git", "config", "user.email", "*****@*****.**"])
def add_and_commit(fname):
directory = os.path.dirname(fname)
if directory and not os.path.exists(directory):
os.makedirs(directory)
with open(fname, "w") as fh:
fh.write(fname + "content")
runner.run(["git", "add", fname])
runner.run(["git", "commit", "-m", "add " + fname])
@pytest.fixture(scope=scope)
def fixture():
# We can't use pytest's tempdir because that is limited to
# scope=function.
tmpdir = tempfile.mkdtemp(prefix="reproman-tests-")
repodir = os.path.realpath(os.path.join(tmpdir, "repo0"))
os.mkdir(repodir)
retval = repodir
with chpwd(repodir):
runner.run(["git", "init"])
setup_user()
if kind != "empty":
add_and_commit("foo")
add_and_commit("bar")
runner.run(["git", "tag", "tag0"])
add_and_commit("subdir/baz")
if kind == "pair":
localdir = os.path.realpath(os.path.join(tmpdir, "repo1"))
runner.run(["git", "clone", repodir, localdir],
expect_stderr=True)
with chpwd(localdir):
setup_user()
retval = localdir, repodir
yield retval
shutil.rmtree(tmpdir)
return fixture
def fixture():
skip_if_no_svn()
repo_name = 'svnrepo'
tmpdir = os.path.realpath(tempfile.mkdtemp(prefix='reproman-tests-'))
root_dir = os.path.join(tmpdir, repo_name)
subdir = os.path.join(tmpdir, 'subdir')
os.mkdir(subdir)
runner = Runner()
runner.run(['svnadmin', 'create', root_dir])
runner.run(['svn', 'checkout', 'file://' + root_dir], cwd=subdir)
checked_out_dir = os.path.join(subdir, repo_name)
if kind != 'empty':
runner.run(['touch', 'foo'], cwd=checked_out_dir)
runner.run(['svn', 'add', 'foo'], cwd=checked_out_dir)
runner.run(['svn', 'commit', '-m', 'bar'], cwd=checked_out_dir)
yield (root_dir, checked_out_dir)
shutil.rmtree(tmpdir)
def venv_test_dir():
skipif.no_network()
dirs = AppDirs('reproman')
test_dir = os.path.join(dirs.user_cache_dir, 'venv_test')
if os.path.exists(test_dir):
return test_dir
os.makedirs(test_dir)
pymod_dir = os.path.join(test_dir, "minimal_pymodule")
create_pymodule(pymod_dir)
runner = Runner(cwd=test_dir)
pip0 = op.join("venv0", "bin", "pip")
pip1 = op.join("venv1", "bin", "pip")
runner.run(["virtualenv", "--python", PY_VERSION, "venv0"])
runner.run(["virtualenv", "--python", PY_VERSION, "venv1"])
runner.run([pip0, "install", "pyyaml"])
runner.run([pip0, "install", "-e", pymod_dir])
runner.run([pip1, "install", "attrs"])
# Make sure we're compatible with older pips.
runner.run([pip1, "install", "pip==9.0.3"])
return test_dir
def open(self):
self._runner = Runner()
class ShellSession(POSIXSession):
"""Local shell session"""
def __init__(self):
super(ShellSession, self).__init__()
self._runner = None
@borrowdoc(Session)
def open(self):
self._runner = Runner()
@borrowdoc(Session)
def close(self):
self._runner = None
@borrowdoc(Session)
def _execute_command(self, command, env=None, cwd=None, with_shell=False):
# XXX should it be a generic behavior to auto-start?
if self._runner is None:
self.open()
run_kw = {}
if env:
# if anything custom, then we need to get original full environment
# and update it with custom settings which we either "accumulated"
# via set_envvar, or it was passed into this call.
run_kw['env'] = get_updated_env(os.environ, env)
return self._runner.run(
command,
# For now we do not ERROR out whenever command fails or provides
# stderr -- analysis will be done outside
expect_fail=True,
expect_stderr=True,
cwd=cwd,
**run_kw) # , shell=True)
@borrowdoc(Session)
def isdir(self, path):
return os.path.isdir(path)
@borrowdoc(Session)
def mkdir(self, path, parents=False):
if not os.path.exists(path):
if parents:
os.makedirs(path)
else:
try:
os.mkdir(path)
except OSError:
raise CommandError(
msg="Failed to make directory {}".format(path))
@borrowdoc(Session)
def get(self, src_path, dest_path=None, uid=-1, gid=-1):
dest_path = self._prepare_dest_path(src_path, dest_path)
if os.path.isdir(src_path):
shutil.copytree(src_path, dest_path)
else:
shutil.copy(src_path, dest_path)
if uid > -1 or gid > -1:
self.chown(dest_path, uid, gid, recursive=True)
@borrowdoc(Session)
def put(self, src_path, dest_path, uid=-1, gid=-1):
# put is the same as get for the shell resource
self.get(src_path, dest_path, uid, gid)
def is_running():
try:
Runner().run(["condor_status"])
except CommandError as exc:
return False
return True
def venv_test_dir():
skipif.no_network()
dirs = AppDirs('reproman')
ssp = os.getenv("REPROMAN_TESTS_ASSUME_SSP")
# Encode the SSP value in the directory name so that the caller doesn't
# have to worry about deleting the cached venvs before setting the flag..
test_dir = os.path.join(dirs.user_cache_dir,
'venv_test{}'.format("_ssp" if ssp else ""))
if os.path.exists(test_dir):
return test_dir
os.makedirs(test_dir)
pymod_dir = os.path.join(test_dir, "minimal_pymodule")
create_pymodule(pymod_dir)
runner = Runner(cwd=test_dir)
pip0 = op.join("venv0", "bin", "pip")
pip1 = op.join("venv1", "bin", "pip")
runner.run(["virtualenv", "--python", PY_VERSION, "venv0"])
runner.run(["virtualenv", "--python", PY_VERSION, "venv1"])
runner.run([pip0, "install", "pyyaml"])
runner.run([pip0, "install", "-e", pymod_dir])
runner.run([pip1, "install", "attrs"])
# Make sure we're compatible with older pips.
runner.run([pip1, "install", "pip==9.0.3"])
if ssp:
# The testing environment supports --system_site_packages.
pip2 = op.join("venv-nonlocal", "bin", "pip")
runner.run([
"virtualenv", "--python", PY_VERSION, "--system-site-packages",
"venv-nonlocal"
])
runner.run([pip2, "install", pymod_dir])
return test_dir
def __cmp__(self, other):
if other is self:
return 0
raise TypeError("UNKNOWN version is not comparable")
#
# Custom handlers
#
from reproman.cmd import Runner
from reproman.support.exceptions import (
MissingExternalDependency,
OutdatedExternalDependency,
)
_runner = Runner()
def _get_annex_version():
"""Return version of available git-annex"""
return _runner.run('git annex version --raw'.split())[0]
def _get_git_version():
"""Return version of available git"""
return _runner.run('git version'.split())[0].split()[-1]
def _get_apt_cache_version():
"""Return version of available apt-cache."""
return _runner.run('apt-cache -v'.split())[0].split()[1]
