def docker_fixture():
"""The actual fixture code generated by get_docker_fixture
on setup, this fixture ensures that a docker container is running
and starts one if necessary.
Fixture yields parameters of the container with a `custom` field passed
into the `get_docker_container`.
on teardown, this fixture stops the docker container it started
"""
skip_if_no_network()
args = [
'docker',
'run',
'-d',
'--rm',
]
params = {}
if name:
args += ['--name', name]
params['name'] = name
if portmaps:
for from_to in portmaps.items():
args += ['-p', '%d:%d' % from_to]
params['port'] = from_to[0]
args += [image]
stdout, _ = Runner().run(args)
params['container_id'] = container_id = stdout.strip()
params['custom'] = custom_params
yield params
Runner().run(['docker', 'stop', container_id])
def check_run_and_get_output(cmd):
runner = Runner()
try:
# suppress log output happen it was set to high values
with patch.dict('os.environ', {'NICEMAN_LOGLEVEL': 'WARN'}):
output = runner.run(["niceman", "--help"])
except CommandError as e:
raise AssertionError("'niceman --help' failed to start normally. "
"Exited with %d and output %s" % (e.code, (e.stdout, e.stderr)))
return output
def test_create_and_start(tmpdir):
runner = Runner()
tmpdir = str(tmpdir)
cfg_file = op.join(tmpdir, "custom.cfg")
inventory_file = op.join(tmpdir, "inventory.yml")
with open(cfg_file, "w") as cfg_fh:
cfg_fh.write("[general]\ninventory_file = {}\n".format(inventory_file))
def run_niceman(args):
runner(["niceman", "--config", cfg_file] + args, expect_stderr=True)
run_niceman(["create", "--resource-type=shell", "myshell"])
with open(inventory_file) as ifh:
dumped = ifh.read()
assert "myshell" in dumped
assert "id" in dumped
# Running with a different config fails ...
empty_cfg_file = op.join(tmpdir, "empty.cfg")
with open(empty_cfg_file, "w"):
pass
with swallow_logs(new_level=logging.ERROR) as cml:
with pytest.raises(CommandError):
runner(["niceman", "--config", empty_cfg_file, "start", "myshell"])
if os.environ.get("NICEMAN_LOGTARGET", "stderr") == "stderr":
assert "ResourceNotFoundError" in cml.out
# ... but using the same config works.
run_niceman(["start", "myshell"])
def revision(self):
# svn info doesn't give the current revision
# (see http://svnbook.red-bean.com/en/1.7/svn.tour.history.html)
# so we need to run svn log
if not hasattr(self, '_revision'):
runner = Runner()
log = runner(['svn', 'log', '^/', '-l', '1'], cwd=self.path)[0]
lines = log.strip().split('\n')
if len(lines) == 1:
self._revision = None
else:
revision = lines[1].split()[0]
if revision.startswith('r') and revision[1:].isdigit():
self._revision = int(revision[1:])
else:
self._revision = revision[1:]
return self._revision
def test_git_repo_detached(git_repo):
runner = Runner()
# If we are in a detached state, we still don't identify the
# repository itself.
runner(["git", "checkout", "master^{}", "--"],
cwd=git_repo,
expect_stderr=True)
hexsha_master, _ = runner(["git", "rev-parse", "master"], cwd=git_repo)
hexsha_master = hexsha_master.strip()
tracer = VCSTracer()
dists = list(tracer.identify_distributions([git_repo]))
pkg = dists[0][0].packages[0]
# We do not include repository path itself.
assert pkg.files == []
assert pkg.hexsha == hexsha_master
assert not pkg.branch
assert not pkg.remotes
def chown(self, path, uid=-1, gid=-1, recursive=False, remote=True):
"""Set the user and gid of a path
"""
uid = int(uid) # Command line parameters getting passed as type str
gid = int(gid)
if uid == -1 and gid > -1:
command = ['chgrp']
else:
command = ['chown']
if recursive: command += ["-R"]
if uid > -1 and gid > -1: command += ["{}.{}".format(uid, gid)]
elif uid > -1: command += [uid]
elif gid > -1: command += [gid]
else:
raise CommandError(cmd='chown',
msg="Invalid command \
parameters.")
command += [path]
if remote:
self.execute_command(command)
else:
# Run on the local file system
Runner().run(command)
def venv_test_dir():
dirs = AppDirs('niceman')
test_dir = os.path.join(dirs.user_cache_dir, 'venv_test')
if os.path.exists(test_dir):
return test_dir
runner = Runner()
runner.run(["mkdir", "-p", test_dir])
pymod_dir = os.path.join(test_dir, "minimal_pymodule")
create_pymodule(pymod_dir)
with chpwd(test_dir):
runner.run(["virtualenv", "--python", PY_VERSION, "venv0"])
runner.run(["virtualenv", "--python", PY_VERSION, "venv1"])
runner.run(["./venv0/bin/pip", "install", "pyyaml"])
runner.run(["./venv0/bin/pip", "install", "-e", pymod_dir])
runner.run(["./venv1/bin/pip", "install", "attrs"])
return test_dir
def start(self):
self._runner = Runner()
class ShellSession(POSIXSession):
"""Local shell session"""
def __init__(self):
super(ShellSession, self).__init__()
self._runner = None
def start(self):
self._runner = Runner()
def stop(self):
self._runner = None
#
# Commands fulfilling a "Session" interface to interact with the environment
#
def _execute_command(self, command, env=None, cwd=None):
"""
Execute the given command in the environment.
Parameters
----------
command : list
Shell command string or list of command tokens to send to the
environment to execute.
env : dict
Additional (or replacement) environment variables which are applied
only to the current call
Returns
-------
out, err
"""
# XXX should it be a generic behavior to auto-start?
if self._runner is None:
self.start()
run_kw = {}
if env:
# if anything custom, then we need to get original full environment
# and update it with custom settings which we either "accumulated"
# via set_envvar, or it was passed into this call.
run_kw['env'] = get_updated_env(os.environ, env)
return self._runner.run(
command,
# For now we do not ERROR out whenever command fails or provides
# stderr -- analysis will be done outside
expect_fail=True,
expect_stderr=True,
cwd=cwd,
**run_kw
) # , shell=True)
def isdir(self, path):
return os.path.isdir(path)
def mkdir(self, path, parents=False):
if not os.path.exists(path):
if parents:
os.makedirs(path)
else:
os.mkdir(path)
def git_repo_fixture(kind="default", scope="function"):
"""Create a Git repository fixture.
Parameters
----------
kind : {"empty", "default", "pair"}, optional
Kind git repository.
- empty: a repository with no commits.
- default: a repository with three commits, each adding one of
its three files: "foo", "bar", and "subdir/baz".
- pair: a (local, remote) pair of Git repos. The repos have
the same structure as the "default" repo, but "local" has a
remote "origin" with a URL that points to "remote".
scope : {"function", "class", "module", "session"}, optional
A `pytest.fixture` scope argument.
Returns
-------
A fixture function.
"""
runner = Runner()
def setup_user():
# Set the user in the local Git configuration rather than
# through environmental variables so that test functions using
# this fixture can make commits under the same user.
runner(["git", "config", "user.name", "A U Thor"])
runner(["git", "config", "user.email", "*****@*****.**"])
def add_and_commit(fname):
directory = os.path.dirname(fname)
if directory and not os.path.exists(directory):
os.makedirs(directory)
with open(fname, "w") as fh:
fh.write(fname + "content")
runner.run(["git", "add", fname])
runner.run(["git", "commit", "-m", "add " + fname])
@pytest.fixture(scope=scope)
def fixture():
# We can't use pytest's tempdir because that is limited to
# scope=function.
tmpdir = tempfile.mkdtemp(prefix="niceman-tests-")
repodir = os.path.realpath(os.path.join(tmpdir, "repo0"))
os.mkdir(repodir)
retval = repodir
with chpwd(repodir):
runner.run(["git", "init"])
setup_user()
if kind != "empty":
add_and_commit("foo")
add_and_commit("bar")
runner.run(["git", "tag", "tag0"])
add_and_commit("subdir/baz")
if kind == "pair":
localdir = os.path.realpath(os.path.join(tmpdir, "repo1"))
runner.run(["git", "clone", repodir, localdir],
expect_stderr=True)
with chpwd(localdir):
setup_user()
retval = localdir, repodir
yield retval
shutil.rmtree(tmpdir)
return fixture
def fixture():
skip_if_no_svn()
repo_name = 'svnrepo'
tmpdir = os.path.realpath(tempfile.mkdtemp(prefix='niceman-tests-'))
root_dir = os.path.join(tmpdir, repo_name)
subdir = os.path.join(tmpdir, 'subdir')
os.mkdir(subdir)
runner = Runner()
runner.run(['svnadmin', 'create', root_dir])
runner.run(['svn', 'checkout', 'file://' + root_dir], cwd=subdir)
checked_out_dir = os.path.join(subdir, repo_name)
if kind != 'empty':
runner.run(['touch', 'foo'], cwd=checked_out_dir)
runner.run(['svn', 'add', 'foo'], cwd=checked_out_dir)
runner.run(['svn', 'commit', '-m', 'bar'], cwd=checked_out_dir)
yield (root_dir, checked_out_dir)
shutil.rmtree(tmpdir)
def open(self):
self._runner = Runner()
class ShellSession(POSIXSession):
"""Local shell session"""
def __init__(self):
super(ShellSession, self).__init__()
self._runner = None
@borrowdoc(Session)
def open(self):
self._runner = Runner()
@borrowdoc(Session)
def close(self):
self._runner = None
@borrowdoc(Session)
def _execute_command(self, command, env=None, cwd=None):
# XXX should it be a generic behavior to auto-start?
if self._runner is None:
self.open()
run_kw = {}
if env:
# if anything custom, then we need to get original full environment
# and update it with custom settings which we either "accumulated"
# via set_envvar, or it was passed into this call.
run_kw['env'] = get_updated_env(os.environ, env)
return self._runner.run(
command,
# For now we do not ERROR out whenever command fails or provides
# stderr -- analysis will be done outside
expect_fail=True,
expect_stderr=True,
cwd=cwd,
**run_kw) # , shell=True)
@borrowdoc(Session)
def isdir(self, path):
return os.path.isdir(path)
@borrowdoc(Session)
def mkdir(self, path, parents=False):
if not os.path.exists(path):
if parents:
os.makedirs(path)
else:
try:
os.mkdir(path)
except OSError:
raise CommandError(
msg="Failed to make directory {}".format(path))
@borrowdoc(Session)
def get(self, src_path, dest_path=None, uid=-1, gid=-1):
dest_path = self._prepare_dest_path(src_path, dest_path)
shutil.copy(src_path, dest_path)
if uid > -1 or gid > -1:
self.chown(dest_path, uid, gid, recursive=True)
@borrowdoc(Session)
def put(self, src_path, dest_path, uid=-1, gid=-1):
# put is the same as get for the shell resource
self.get(src_path, dest_path, uid, gid)
"""For internal use
"""
def __str__(self):
return "UNKNOWN"
def __cmp__(self, other):
if other is self:
return 0
raise TypeError("UNKNOWN version is not comparable")
#
# Custom handlers
#
from niceman.cmd import Runner
_runner = Runner()
def _get_annex_version():
"""Return version of available git-annex"""
return _runner.run('git annex version --raw'.split())[0]
def _get_git_version():
"""Return version of available git"""
return _runner.run('git version'.split())[0].split()[-1]
def _get_apt_cache_version():
"""Return version of available git"""
return _runner.run('apt-cache -v'.split())[0].split()[1]
def test_detached_git(repo=None):
import os
# XXX Replace with our session?
from niceman.cmd import Runner
env = os.environ.copy()
env['LC_ALL'] = 'C'
runner = Runner(env=env, cwd=repo)
assert runner('git config user.name')[0], "git env should be set"
assert runner('git config user.email')[0], "git env should be set"
runner('git init')
# should be good enough not to crash
distributions, files = identify_distributions([repo])
assert len(distributions) == 1
dist = distributions[0]
assert dist.name == 'git'
packages = dist.packages
assert len(packages) == 1
pkg = packages[0]
# we do not include repository path itself
assert pkg.files == []
assert pkg.path == repo
# Let's now make it more sensible
fname = opj(repo, "file")
with open(fname, 'w') as f:
f.write("data")
runner("git add file")
runner("git commit -m added file")
distributions, files = identify_distributions([fname])
assert len(distributions) == 1
pkg = distributions[0].packages[0]
assert pkg.files == [fname]
hexsha = pkg.hexsha
assert hexsha
assert pkg.branch == 'master'
# And if point to a directory under, should not identify the VCS
# (since in principle git e.g. is not tracing directories)
subdir = opj(repo, 'subdir')
os.mkdir(subdir)
distributions_, files_ = identify_distributions([subdir])
assert distributions_ == []
assert files_ == [subdir]
# but if we point to a file under
subfile = opj(subdir, 'file')
with open(subfile, 'w') as f:
pass
runner("git add subdir/file")
distributions__, files__ = identify_distributions([subfile])
assert len(distributions__) == 1
pkg = distributions__[0].packages[0]
assert pkg.files == [subfile]
# and if we cause a detachment
runner("git rm file")
runner("git commit -m removed file")
runner("git checkout HEAD^", expect_stderr=True)
distributions, files = identify_distributions([repo])
pkg = distributions[0].packages[0]
# we do not include repository path itself
assert pkg.files == []
assert pkg.hexsha == hexsha
assert not pkg.branch
assert not pkg.remotes
def test_git_repo_remotes(git_repo_pair):
repo_local, repo_remote = git_repo_pair
runner = Runner()
tracer = VCSTracer()
# Set remote.pushdefault to a value we know doesn't exist.
# Otherwise, the test machine may have remote.pushdefault globally
# configured to point to "origin".
runner.run(["git", "config", "remote.pushdefault", "notexisting"],
cwd=repo_local)
# Add another remote that doesn't contain the current commit (in
# fact doesn't actually exist), so that we test the "listed as
# remote but doesn't contain" case.
runner.run(["git", "remote", "add", "fakeremote", "fakepath"],
cwd=repo_local)
paths = [os.path.join(repo_local, "foo")]
dists_nopush = list(tracer.identify_distributions(paths))
assert_distributions(dists_nopush,
expected_length=1,
expected_subset={
"name":
"git",
"packages": [{
"files": paths,
"path": repo_local,
"branch": "master",
"tracked_remote": "origin",
"remotes": {
"origin": {
"url": repo_remote,
"contains": True
},
"fakeremote": {
"url": "fakepath"
}
}
}]
})
pkg_nopush = dists_nopush[0][0].packages[0]
assert set(pkg_nopush.remotes.keys()) == {"origin", "fakeremote"}
# fakeremote, which doesn't contain the current commit, doesn't
# have contains=True.
assert "contains" in pkg_nopush.remotes["origin"]
assert "contains" not in pkg_nopush.remotes["fakeremote"]
# pushurl is not included in the output above because it is not
# set.
assert "pushurl" not in list(pkg_nopush.remotes.values())
# If we set the pushurl and retrace, it is included.
runner.run(["git", "config", "remote.origin.pushurl", repo_remote],
cwd=repo_local)
dists_push = list(tracer.identify_distributions(paths))
pkg_push = dists_push[0][0].packages[0]
assert pkg_push.remotes["origin"]["pushurl"] == repo_remote
# If we are at a commit that none of the remotes are known to
# contain, there are no listed remotes.
with chpwd(repo_local):
runner(["git", "commit", "--allow-empty", "-m", "empty commit"])
dists_nocontain = list(tracer.identify_distributions(paths))
assert not dists_nocontain[0][0].packages[0].remotes.values()
# The remote repository, however, doesn't have a remote, so there
# are not listed remotes.
paths = [os.path.join(repo_remote, "foo")]
dists_remote = list(tracer.identify_distributions(paths))
assert not dists_remote[0][0].packages[0].remotes.values()