def test_distributions(demo1_spec):
provenance = Provenance.factory(demo1_spec)
distributions = provenance.get_distributions()
distributions = items_to_dict(distributions)
assert set(distributions.keys()) == {'conda', 'debian'}
# Test DebianDistribution class.
debian_distribution = distributions['debian']
environment = MagicMock()
with swallow_logs(new_level=logging.DEBUG) as log:
debian_distribution.initiate(environment)
debian_distribution.install_packages(environment)
calls = [
call.execute_command(['apt-get', 'update']),
call.execute_command([
'apt-get', 'install', '-y', 'libc6-dev=2.19-18+deb8u4',
'afni=16.2.07~dfsg.1-2~nd90+1'
]),
]
environment.assert_has_calls(calls, any_order=True)
assert_in("Adding Debian update to environment command list.",
log.lines)
"""
def test_globbedpaths(path=None):
dotdir = op.curdir + op.sep
for patterns, expected in [
(["1.txt", "2.dat"], {"1.txt", "2.dat"}),
([dotdir + "1.txt", "2.dat"], {dotdir + "1.txt", "2.dat"}),
(["*.txt", "*.dat"], {"1.txt", "2.dat", u"bβ.dat", "3.txt"}),
([dotdir + "*.txt", "*.dat"],
{dotdir + "1.txt", "2.dat", u"bβ.dat", dotdir + "3.txt"}),
(["subdir/*.txt"], {"subdir/1.txt", "subdir/2.txt"}),
([dotdir + "subdir/*.txt"],
{dotdir + p for p in ["subdir/1.txt", "subdir/2.txt"]}),
(["*.txt"], {"1.txt", "3.txt"})]:
gp = GlobbedPaths(patterns, pwd=path)
eq_(set(gp.expand()), expected)
eq_(set(gp.expand(full=True)),
{op.join(path, p) for p in expected})
pardir = op.pardir + op.sep
subdir_path = op.join(path, "subdir")
for patterns, expected in [
(["*.txt"], {"1.txt", "2.txt"}),
([dotdir + "*.txt"], {dotdir + p for p in ["1.txt", "2.txt"]}),
([pardir + "*.txt"], {pardir + p for p in ["1.txt", "3.txt"]}),
([dotdir + pardir + "*.txt"],
{dotdir + pardir + p for p in ["1.txt", "3.txt"]}),
(["subdir/"], {"subdir/"})]:
gp = GlobbedPaths(patterns, pwd=subdir_path)
eq_(set(gp.expand()), expected)
eq_(set(gp.expand(full=True)),
{op.join(subdir_path, p) for p in expected})
# Full patterns still get returned as relative to pwd.
gp = GlobbedPaths([op.join(path, "*.dat")], pwd=path)
eq_(gp.expand(), ["2.dat", u"bβ.dat"])
# "." gets special treatment.
gp = GlobbedPaths([".", "*.dat"], pwd=path)
eq_(set(gp.expand()), {"2.dat", u"bβ.dat", "."})
eq_(gp.expand(dot=False), ["2.dat", u"bβ.dat"])
gp = GlobbedPaths(["."], pwd=path, expand=False)
eq_(gp.expand(), ["."])
eq_(gp.paths, ["."])
# We can the glob outputs.
glob_results = {"z": "z",
"a": ["x", "d", "b"]}
with patch('glob.glob', glob_results.get):
gp = GlobbedPaths(["z", "a"])
eq_(gp.expand(), ["z", "b", "d", "x"])
# glob expansion for paths property is determined by expand argument.
for expand, expected in [(True, ["2.dat", u"bβ.dat"]),
(False, ["*.dat"])]:
gp = GlobbedPaths(["*.dat"], pwd=path, expand=expand)
eq_(gp.paths, expected)
with swallow_logs(new_level=logging.DEBUG) as cml:
GlobbedPaths(["not here"], pwd=path).expand()
assert_in("No matching files found for 'not here'", cml.out)
def test_retrace(reprozip_spec2):
"""
Test installing packages on the localhost.
"""
with swallow_logs(new_level=logging.DEBUG) as log:
args = [
'retrace',
'--spec',
reprozip_spec2,
]
main(args)
assert_in("reading spec file " + reprozip_spec2, log.lines)
def test_delete_interface(niceman_cfg_path):
"""
Test deleting a resource.
"""
with patch('docker.Client') as client, \
patch('niceman.resource.ResourceManager.set_inventory'), \
patch('niceman.resource.ResourceManager.get_inventory') as get_inventory, \
swallow_logs(new_level=logging.DEBUG) as log:
client.return_value = MagicMock(
containers=lambda all: [{
'Id': '326b0fdfbf838',
'Names': ['/my-resource'],
'State': 'running'
}])
get_inventory.return_value = {
"my-resource": {
"status": "running",
"engine_url": "tcp://127.0.0.1:2375",
"type": "docker-container",
"name": "my-resource",
"id": "326b0fdfbf838"
}
}
args = [
'delete', '--name', 'my-resource', '--config', niceman_cfg_path,
'--skip-confirmation'
]
main(args)
calls = [
call(base_url='tcp://127.0.0.1:2375'),
call().remove_container(
{
'State': 'running',
'Id': '326b0fdfbf838',
'Names': ['/my-resource']
},
force=True)
]
client.assert_has_calls(calls, any_order=True)
assert_in('Deleted the environment my-resource', log.lines)
def test_login_interface():
"""
Test logging into an environment
"""
with patch('docker.Client') as client, \
patch('niceman.resource.ResourceManager._get_inventory') as get_inventory, \
patch('dockerpty.start'), \
swallow_logs(new_level=logging.DEBUG) as log:
client.return_value = MagicMock(
containers=lambda all: [{
'Id': '18b31b30e3a5',
'Names': ['/my-test-resource'],
'State': 'running'
}], )
get_inventory.return_value = {
"my-test-resource": {
"status": "running",
"engine_url": "tcp://127.0.0.1:2375",
"type": "docker-container",
"name": "my-test-resource",
"id": "18b31b30e3a5"
}
}
args = ['login', 'my-test-resource']
with patch("niceman.interface.login.get_manager",
return_value=ResourceManager()):
main(args)
assert client.call_count == 1
calls = [call(base_url='tcp://127.0.0.1:2375')]
client.assert_has_calls(calls, any_order=True)
assert_in("Opening TTY connection to docker container.", log.lines)
def test_diff_files():
with swallow_outputs() as outputs:
args = ['diff', diff_1_yaml, diff_2_yaml]
main(args)
assert_equal(outputs.err, '')
assert_in('Files:', outputs.out)
assert_in('< /etc/a', outputs.out)
assert_in('> /etc/c', outputs.out)
assert_not_in('< /etc/b', outputs.out)
assert_not_in('> /etc/b', outputs.out)
def test_create_interface(niceman_cfg_path):
"""
Test creating an environment
"""
with patch('docker.Client') as client, \
patch('niceman.resource.ResourceManager.set_inventory'), \
patch('niceman.resource.ResourceManager.get_inventory') as get_inventory, \
swallow_logs(new_level=logging.DEBUG) as log:
client.return_value = MagicMock(
containers=lambda all: [],
pull=lambda repository, stream: [
b'{ "status" : "status 1", "progress" : "progress 1" }',
b'{ "status" : "status 2", "progress" : "progress 2" }'
],
create_container=lambda name, image, stdin_open, tty, command:
{'Id': '18b31b30e3a5'})
get_inventory.return_value = {
"my-test-resource": {
"status": "running",
"engine_url": "tcp://127.0.0.1:2375",
"type": "docker-container",
"name": "my-test-resource",
"id": "18b31b30e3a5"
}
}
args = [
'create', '--name', 'my-test-resource', '--resource-type',
'docker-container', '--config', niceman_cfg_path, '--backend',
'engine_url=tcp://127.0.0.1:2376'
]
main(args)
calls = [
call(base_url='tcp://127.0.0.1:2376'),
call().start(container='18b31b30e3a5')
]
client.assert_has_calls(calls, any_order=True)
assert_in("status 1 progress 1", log.lines)
assert_in("status 2 progress 2", log.lines)
assert_in("Created the environment my-test-resource", log.lines)
def test_distributions(demo1_spec):
def mock_execute_command(command, env=None):
if isinstance(command, list):
if command == ['apt-cache', 'policy', 'libc6-dev:amd64']:
return (b'libc6-dev: \
Installed: (none) \
Candidate: 2.19-18+deb8u4 \
Version table: \
2.19-18+deb8u4 500 \
500 http://archive.ubuntu.com/ubuntu xenial/universe amd64 Packages',
0)
if command == ['apt-cache', 'policy', 'afni:amd64']:
return (b'afni: \
Installed: 16.2.07~dfsg.1-2~nd90+1 \
Candidate: 16.2.07~dfsg.1-2~nd90+1 \
Version table: \
16.2.07~dfsg.1-2~nd90+1 500 \
500 http://archive.ubuntu.com/ubuntu xenial/universe amd64 Packages',
0)
if command == ['apt-cache', 'policy', 'dcm2niix:amd64']:
return (b'dcm2niix: \
Installed: (none) \
Candidate: 1:1.0.20171017+git3-g9ccc4c0-1~nd16.04+1 \
Version table: \
1:1.0.20171017+git3-g9ccc4c0-1~nd16.04+1 500 \
500 http://archive.ubuntu.com/ubuntu xenial/universe amd64 Packages',
0)
if isinstance(command, str):
if command.startswith('grep'):
return (None, 1)
provenance = Provenance.factory(demo1_spec)
distributions = provenance.get_distributions()
distributions = items_to_dict(distributions)
assert set(distributions.keys()) == {'conda', 'debian'}
# Test DebianDistribution class.
debian_distribution = distributions['debian']
environment = MagicMock()
environment.execute_command = mock_execute_command
environment.exists.return_value = False
with patch('requests.get') as requests, \
swallow_logs(new_level=logging.DEBUG) as log:
requests.return_value = type("TestObject", (object, ), {})()
requests.return_value.text = '<a href="/archive/debian/20171208T032012Z/dists/sid/">next change</a>'
debian_distribution.initiate(environment)
debian_distribution.install_packages(environment)
assert_in("Adding Debian update to environment command list.",
log.lines)
assert_in(
"Adding line 'deb http://snapshot.debian.org/archive/debian/20170531T084046Z/ sid main \
contrib non-free' to /etc/apt/sources.list.d/niceman.sources.list", log.lines)
assert_in(
"Adding line 'deb http://snapshot-neuro.debian.net:5002/archive/neurodebian/20171208T032012Z/ \
xenial main contrib non-free' to /etc/apt/sources.list.d/niceman.sources.list",
log.lines)
assert_in(
"Adding line 'deb http://snapshot-neuro.debian.net:5002/archive/neurodebian/20171208T032012Z/ \
xenial main contrib non-free' to /etc/apt/sources.list.d/niceman.sources.list",
log.lines)
assert_in(
'Installing libc6-dev=2.19-18+deb8u4, afni=16.2.07~dfsg.1-5~nd16.04+1, \
dcm2niix=1:1.0.20171017+git3-g9ccc4c0-1~nd16.04+1', log.lines)
"""
def test_diff_no_distributions():
with swallow_outputs() as outputs:
args = ['diff', diff_1_yaml, empty_yaml]
main(args)
assert_equal(outputs.err, '')
assert_in('Debian packages:', outputs.out)
assert_in('< lib1only x86 2:1.6.4-3', outputs.out)
assert_in('< libsame x86 2.4.6', outputs.out)
assert_in('< libarchdiff x86 2.4.6', outputs.out)
assert_in('< libversdiff x86 2.4.6', outputs.out)
assert_in('Conda packages:', outputs.out)
assert_in('< c_lib1only py36_0 2:1.6.4-3', outputs.out)
assert_in('< c_libsame py36_0 2.4.6', outputs.out)
assert_in('< c_libbuilddiff py36_0 2.4.6', outputs.out)
assert_in('< c_libversdiff py36_0 2.4.6', outputs.out)
assert_in('Files:', outputs.out)
assert_in('< /etc/a', outputs.out)
assert_in('< /etc/b', outputs.out)
def test_diff_conda_packages():
with swallow_outputs() as outputs:
args = ['diff', diff_1_yaml, diff_2_yaml]
main(args)
assert_equal(outputs.err, '')
assert_in('Conda packages:', outputs.out)
assert_in('< c_lib1only py36_0 2:1.6.4-3', outputs.out)
assert_in('> c_lib2only py36_0 2:1.6.4-3', outputs.out)
assert_in('< c_libbuilddiff py36_0 2.4.6', outputs.out)
assert_in('> c_libbuilddiff hdf63c60_3 2.4.6', outputs.out)
# TO DO: ensure the version strings (second and third lines below)
# come from the conda report -- these could just match the debian
# output checked in test_diff_debian_packages()
assert_in('Conda package c_libversdiff py36_0:', outputs.out)
assert_in('< 2.4.6', outputs.out)
assert_in('> 2.4.7', outputs.out)
assert_not_in('c_libsame', outputs.out)
def test_diff_debian_packages():
with swallow_outputs() as outputs:
args = ['diff', diff_1_yaml, diff_2_yaml]
main(args)
assert_equal(outputs.err, '')
assert_in('Debian packages:', outputs.out)
assert_in('< lib1only x86 2:1.6.4-3', outputs.out)
assert_in('> lib2only x86 2:1.6.4-3', outputs.out)
assert_in('< libarchdiff x86 2.4.6', outputs.out)
assert_in('> libarchdiff amd64 2.4.6', outputs.out)
assert_in('Debian package libversdiff x86:', outputs.out)
assert_in('< 2.4.6', outputs.out)
assert_in('> 2.4.7', outputs.out)
assert_not_in('libsame', outputs.out)
def test_diff_svn():
with swallow_outputs() as outputs:
args = ['diff', diff_1_yaml, diff_2_yaml]
main(args)
assert_equal(outputs.err, '')
assert_in('SVN repositories:', outputs.out)
assert_in('< c8ed47ab-45c9-818d-5d62-549dcc6d97d4 (/path/to/svn/repo/1/only)', outputs.out)
assert_in('> d7192e3a-60de-5caa-ccdc9525dea75aabf (/path/to/svn/repo/2/only)', outputs.out)
assert_in('SVN repository 95e4b738-84c7-154c-f082-34d40e21fdd4', outputs.out)
assert_in('< 12 (/path/1/to/different/svn/commit)', outputs.out)
assert_in('> 14 (/path/2/to/different/svn/commit)', outputs.out)
assert_not_in('/path/1/to/common/svn/repo', outputs.out)
assert_not_in('/path/2/to/common/svn/repo', outputs.out)
def test_diff_git():
with swallow_outputs() as outputs:
args = ['diff', diff_1_yaml, diff_2_yaml]
main(args)
assert_equal(outputs.err, '')
assert_in('Git repositories:', outputs.out)
assert_in('< 43e8e6577c7bf493ddb01ea7d49bef7dc7a6643b (/path/to/git/repo/1/only)', outputs.out)
assert_in('> 64b1865267891fdd1a45251ca6f32df213dc546e (/path/to/git/repo/2/only)', outputs.out)
assert_in('Git repository 5b8267181f6cae8dc37aeef21ea54171bd932522', outputs.out)
assert_in('< 3e3aaa73a9c0ca061c7679af5fa7318e70f528ac (/path/1/to/different/git/commit)', outputs.out)
assert_in('> 9d199f7fa7e6f691719e0860c5cf81193e815ad5 (/path/2/to/different/git/commit)', outputs.out)
assert_not_in('/path/1/to/common/git/repo', outputs.out)
assert_not_in('/path/2/to/common/git/repo', outputs.out)
assert_not_in('99ac7f69a070077038a9eb9eca61c028db97181d', outputs.out)
assert_not_in('d057b128759d80a47500adba0c4d3e95092bb87f', outputs.out)
