def test_venv_identify_distributions(venv_test_dir):
pydir = "python{v.major}.{v.minor}".format(v=sys.version_info)
paths = ["lib/" + PY_VERSION + "/site-packages/yaml/parser.py",
"lib/" + PY_VERSION + "/site-packages/attr/filters.py"]
with chpwd(venv_test_dir):
path_args = [
# Both full ...
os.path.join(venv_test_dir, "venv0", paths[0]),
# ... and relative paths work.
os.path.join("venv1", paths[1]),
]
path_args.append("/sbin/iptables")
tracer = VenvTracer()
dists = list(tracer.identify_distributions(path_args))
assert len(dists) == 1
distributions, unknown_files = dists[0]
assert unknown_files == {
"/sbin/iptables",
# The editable package was added by VenvTracer as an unknown file.
os.path.join(venv_test_dir, "minimal_pymodule")}
assert len(distributions.environments) == 2
expect = {"environments":
[{"packages": [{"files": [paths[0]], "name": "PyYAML",
"editable": False},
{"files": [], "name": "nmtest",
"editable": True}]},
{"packages": [{"files": [paths[1]], "name": "attrs",
"editable": False}]}]}
assert_is_subset_recur(expect, attr.asdict(distributions), [dict, list])
def test_venv_install(venv_test_dir, tmpdir):
tmpdir = str(tmpdir)
paths = [
"lib/" + PY_VERSION + "/site-packages/yaml/parser.py",
"lib/" + PY_VERSION + "/site-packages/attr/filters.py"
]
tracer = VenvTracer()
dists = list(
tracer.identify_distributions([
op.join(venv_test_dir, "venv0", paths[0]),
op.join(venv_test_dir, "venv1", paths[1])
]))
assert len(dists) == 1
dist = dists[0][0]
assert len(dist.environments) == 2
for idx in range(2):
dist.environments[idx].path = op.join(tmpdir, "venv{}".format(idx))
dist.initiate(None)
dist.install_packages()
dists_installed = list(
tracer.identify_distributions([
op.join(tmpdir, "venv0", paths[0]),
op.join(tmpdir, "venv1", paths[1])
]))
assert len(dists_installed) == 1
dist_installed = dists_installed[0][0]
expect = {
"environments": [{
"packages": [{
"name": "PyYAML",
"editable": False
}]
}, {
"packages": [{
"name": "attrs",
"editable": False
}]
}]
}
assert_is_subset_recur(expect, attr.asdict(dist_installed), [dict, list])
# We don't yet handle editable packages.
assert any(
[p.name == "nmtest" for e in dist.environments for p in e.packages])
assert not any([
p.name == "nmtest" for e in dist_installed.environments
for p in e.packages
])
def test_pip_batched_show():
pkgs = ["pkg0", "pkg1", "pkg2"]
batches = [
("""\
Name: pkg0
Version: 4.1
Home-page: urlwith---three-dashes
Files:
file0
---
Name: pkg1
Version: 17.4.0
Files:
file1""", None, None), # err, exception
("""\
Name: pkg2
Version: 4""", None, None)
]
with mock.patch("niceman.distributions.piputils.execute_command_batch",
return_value=batches):
pkg_entries = list(piputils._pip_batched_show(None, None, pkgs))
expect = [
(
"pkg0",
{
"Name": "pkg0",
"Version": "4.1",
"Files": ["file0"],
# We did not split on the URL's "---".
"Home-page": "urlwith---three-dashes"
}),
("pkg1", {
"Name": "pkg1",
"Version": "17.4.0",
"Files": ["file1"]
}),
("pkg2", {
"Name": "pkg2",
"Version": "4"
})
]
assert_is_subset_recur(expect, pkg_entries, [dict, list])
def assert_distributions(result,
expected_length=None,
which=0,
expected_unknown=None,
expected_subset=None):
"""Wrap common assertions about identified distributions.
Parameters
----------
result : iterable
The result of a tracer's identify_distributions method (in its
original generator form or as a list).
expected_length : int, optional
Expected number of items in `result`.
which : int, optional
Index specifying which distribution from result to consider
for the `expected_unknown` and `expected_subset`
assertions.
expected_unknown : list, optional
Which files should be marked as unknown in distribution
`which`.
expected_subset : dict, optional
This dict is expected to be a subset of the distribution
`which`. The check is done by `assert_is_subset_recur`.
"""
result = list(result)
if expected_length is not None:
assert len(result) == expected_length
dist, unknown_files = result[which]
if expected_unknown is not None:
assert unknown_files == expected_unknown
if expected_subset is not None:
assert_is_subset_recur(expected_subset, attr.asdict(dist),
[dict, list])
def test_conda_manager_identify_distributions(get_conda_test_dir):
# Skip if network is not available (skip_if_no_network fails with fixtures)
test_dir = get_conda_test_dir
files = [os.path.join(test_dir, "miniconda/bin/sqlite3"),
os.path.join(test_dir, "miniconda/envs/mytest/bin/xz"),
os.path.join(test_dir, "miniconda/envs/mytest/lib/python2.7/site-packages/pip/index.py"),
os.path.join(test_dir, "miniconda/envs/mytest/lib/python2.7/site-packages/rpaths.py"),
"/sbin/iptables"]
tracer = CondaTracer()
dists = list(tracer.identify_distributions(files))
assert len(dists) == 1, "Exactly one Conda distribution expected."
(distributions, unknown_files) = dists[0]
assert unknown_files == ["/sbin/iptables"], \
"Exactly one file (/sbin/iptables) should not be discovered."
assert len(distributions.environments) == 2, \
"Two conda environments are expected."
out = {'environments': [{'packages': [{'files': ['bin/sqlite3'],
'name': 'sqlite'}]},
{'packages': [{'files': ['bin/xz'],
'name': 'xz'},
{'files': ['lib/python2.7/site-packages/pip/index.py'],
'name': 'pip'},
{'files': ['lib/python2.7/site-packages/rpaths.py'],
'installer': 'pip',
'name': 'rpaths'}
]
}
]
}
assert_is_subset_recur(out, attr.asdict(distributions), [dict, list])
NicemanProvenance.write(sys.stdout, distributions)
print(json.dumps(unknown_files, indent=4))
def test_parse_apt_cache_show_pkgs_output():
from ..debian import parse_apt_cache_show_pkgs_output
txt1 = """\
Package: openssl
Status: install ok installed
Priority: optional
Section: utils
Installed-Size: 934
Maintainer: Ubuntu Developers <*****@*****.**>
Architecture: amd64
Version: 1.0.2g-1ubuntu4.5
Depends: libc6 (>= 2.15), libssl1.0.0 (>= 1.0.2g)
Suggests: ca-certificates
Conffiles:
/etc/ssl/openssl.cnf 7df26c55291b33344dc15e3935dabaf3
Description-en: Secure Sockets Layer toolkit - cryptographic utility
This package is part of the OpenSSL project's implementation of the SSL
and TLS cryptographic protocols for secure communication over the
Internet.
.
It contains the general-purpose command line binary /usr/bin/openssl,
useful for cryptographic operations such as:
* creating RSA, DH, and DSA key parameters;
* creating X.509 certificates, CSRs, and CRLs;
* calculating message digests;
* encrypting and decrypting with ciphers;
* testing SSL/TLS clients and servers;
* handling S/MIME signed or encrypted mail.
Description-md5: 9b6de2bb6e1d9016aeb0f00bcf6617bd
Original-Maintainer: Debian OpenSSL Team <*****@*****.**>
Package: openssl
Priority: standard
Section: utils
Installed-Size: 934
Maintainer: Ubuntu Developers <*****@*****.**>
Original-Maintainer: Debian OpenSSL Team <*****@*****.**>
Architecture: amd64
Source: openssl-src (1.0.2g)
Version: 1.0.2g-1ubuntu4
Depends: libc6 (>= 2.15), libssl1.0.0 (>= 1.0.2g)
Suggests: ca-certificates
Filename: pool/main/o/openssl/openssl_1.0.2g-1ubuntu4_amd64.deb
Size: 492190
MD5sum: 8280148dc2991da94be5810ad4d91552
SHA1: b5326f27aae83c303ff934121dede47d9fce7c76
SHA256: e897ffc8d84b0d436baca5dbd684a85146ffa78d3f2d15093779d3f5a8189690
Description-en: Secure Sockets Layer toolkit - cryptographic utility
This package is part of the OpenSSL project's implementation of the SSL
and TLS cryptographic protocols for secure communication over the
Internet.
.
It contains the general-purpose command line binary /usr/bin/openssl,
useful for cryptographic operations such as:
* creating RSA, DH, and DSA key parameters;
* creating X.509 certificates, CSRs, and CRLs;
* calculating message digests;
* encrypting and decrypting with ciphers;
* testing SSL/TLS clients and servers;
* handling S/MIME signed or encrypted mail.
Description-md5: 9b6de2bb6e1d9016aeb0f00bcf6617bd
Bugs: https://bugs.launchpad.net/ubuntu/+filebug
Origin: Ubuntu
Supported: 5y
Task: standard, ubuntu-core, ubuntu-core, mythbuntu-frontend, mythbuntu-backend-slave, mythbuntu-backend-master, ubuntu-touch-core, ubuntu-touch, ubuntu-sdk-libs-tools, ubuntu-sdk
Package: alienblaster
Priority: extra
Section: universe/games
Installed-Size: 668
Maintainer: Ubuntu Developers <*****@*****.**>
Original-Maintainer: Debian Games Team <*****@*****.**>
Architecture: amd64
Source: alienblaster-src
Version: 1.1.0-9
Depends: alienblaster-data, libc6 (>= 2.14), libgcc1 (>= 1:3.0), libsdl-mixer1.2, libsdl1.2debian (>= 1.2.11), libstdc++6 (>= 5.2)
Filename: pool/universe/a/alienblaster/alienblaster_1.1.0-9_amd64.deb
Size: 180278
MD5sum: e53379fd0d60e0af6304af78aa8ef2b7
SHA1: ca405056cf66a1c2ae3ae1674c22b7d24cda4986
SHA256: ff25bd843420801e9adea4f5ec1ca9656b2aeb327d8102107bf5ebbdb3046c38
Description-en: Classic 2D shoot 'em up
Your mission is simple: Stop the invasion of the aliens and blast them!
.
Alien Blaster is a classic 2D shoot 'em up featuring lots of different
weapons, special items, aliens to blast and a big bad boss.
.
It supports both a single player mode and a cooperative two player mode
for two persons playing on one computer.
Description-md5: da1f8f1a6453d62874036331e075d65f
Homepage: http://www.schwardtnet.de/alienblaster/
Bugs: https://bugs.launchpad.net/ubuntu/+filebug
Origin: Ubuntu
"""
out1 = {
'alienblaster:amd64=1.1.0-9': {
'Architecture': 'amd64',
'Source_name': 'alienblaster-src',
'Package': 'alienblaster',
'Version': '1.1.0-9'
},
'openssl:amd64=1.0.2g-1ubuntu4': {
'Architecture': 'amd64',
'Source_name': 'openssl-src',
'Source_version': '1.0.2g',
'Package': 'openssl',
'Version': '1.0.2g-1ubuntu4'
},
'openssl:amd64=1.0.2g-1ubuntu4.5': {
'Architecture': 'amd64',
'Package': 'openssl',
'Status': 'install ok installed',
'Version': '1.0.2g-1ubuntu4.5'
}
}
out = parse_apt_cache_show_pkgs_output(txt1)
assert_is_subset_recur(out1, out, [dict])
def test_parse_apt_cache_policy_source_info():
from ..debian import parse_apt_cache_policy_source_info
txt = """\
Package files:
100 /var/lib/dpkg/status
release a=now
500 http://neuro.debian.net/debian xenial/non-free i386 Packages
release o=NeuroDebian,a=xenial,n=xenial,l=NeuroDebian,c=non-free,b=i386
origin neuro.debian.net
500 http://neuro.debian.net/debian xenial/non-free amd64 Packages
release o=NeuroDebian,a=xenial,n=xenial,l=NeuroDebian,c=non-free,b=amd64
origin neuro.debian.net
500 http://neuro.debian.net/debian data/non-free i386 Packages
release o=NeuroDebian,a=data,n=data,l=NeuroDebian,c=non-free,b=i386
origin neuro.debian.net
500 http://neuro.debian.net/debian data/non-free amd64 Packages
release o=NeuroDebian,a=data,n=data,l=NeuroDebian,c=non-free,b=amd64
origin neuro.debian.net
500 file:/my/repo2 ubuntu/ Packages
release c=
500 file:/my/repo ./ Packages
release c=
500 http://dl.google.com/linux/chrome/deb stable/main amd64 Packages
release v=1.0,o=Google, Inc.,a=stable,n=stable,l=Google,c=main,b=amd64
origin dl.google.com
500 http://security.ubuntu.com/ubuntu xenial-security/restricted i386 Packages
release v=16.04,o=Ubuntu,a=xenial-security,n=xenial,l=Ubuntu,c=restricted,b=i386
origin security.ubuntu.com
500 http://security.ubuntu.com/ubuntu xenial-security/restricted amd64 Packages
release v=16.04,o=Ubuntu,a=xenial-security,n=xenial,l=Ubuntu,c=restricted,b=amd64
origin security.ubuntu.com
500 http://debproxy:9999/debian/ jessie-backports/contrib Translation-en
100 http://debproxy:9999/debian/ jessie-backports/non-free amd64 Packages
release o=Debian Backports,a=jessie-backports,n=jessie-backports,l=Debian Backports,c=non-free
origin debproxy
500 http://us.archive.ubuntu.com/ubuntu xenial-updates/universe amd64 Packages
release v=16.04,o=Ubuntu,a=xenial-updates,n=xenial,l=Ubuntu,c=universe,b=amd64
origin us.archive.ubuntu.com
500 http://us.archive.ubuntu.com/ubuntu xenial-updates/multiverse i386 Packages
release v=16.04,o=Ubuntu,a=xenial-updates,n=xenial,l=Ubuntu,c=multiverse,b=i386
origin us.archive.ubuntu.com
Pinned packages:
"""
out1 = {
'http://neuro.debian.net/debian xenial/non-free i386 Packages': {
'architecture': 'i386',
'archive': 'xenial',
'archive_uri': 'http://neuro.debian.net/debian',
'uri_suite': 'xenial',
'codename': 'xenial',
'component': 'non-free',
'label': 'NeuroDebian',
'origin': 'NeuroDebian',
'site': 'neuro.debian.net'
},
'http://security.ubuntu.com/ubuntu xenial-security/restricted amd64 Packages':
{
'architecture': 'amd64',
'archive': 'xenial-security',
'archive_uri': 'http://security.ubuntu.com/ubuntu',
'uri_suite': 'xenial-security',
'codename': 'xenial',
'component': 'restricted',
'label': 'Ubuntu',
'origin': 'Ubuntu',
'site': 'security.ubuntu.com'
},
'http://debproxy:9999/debian/ jessie-backports/contrib Translation-en':
{
'archive_uri': 'http://debproxy:9999/debian/',
'uri_suite': 'jessie-backports'
},
'http://debproxy:9999/debian/ jessie-backports/non-free amd64 Packages':
{
'archive': 'jessie-backports',
'archive_uri': 'http://debproxy:9999/debian/',
'codename': 'jessie-backports',
'component': 'non-free',
'label': 'Debian Backports',
'origin': 'Debian Backports',
'site': 'debproxy',
'uri_suite': 'jessie-backports'
},
}
out = parse_apt_cache_policy_source_info(txt)
assert_is_subset_recur(out1, out, [dict])
def test_parse_apt_cache_policy_pkgs_output():
from ..debian import parse_apt_cache_policy_pkgs_output
txt1 = """\
afni:
Installed: 16.2.07~dfsg.1-2~nd90+1
Candidate: 16.2.07~dfsg.1-2~nd90+1
Version table:
*** 16.2.07~dfsg.1-2~nd90+1 500
500 http://neuro.debian.net/debian stretch/contrib amd64 Packages
100 /var/lib/dpkg/status
openssl:
Installed: 1.0.2g-1ubuntu4.5
Candidate: 1.0.2g-1ubuntu4.8
Version table:
1.0.2g-1ubuntu4.8 500
500 http://us.archive.ubuntu.com/ubuntu xenial-updates/main amd64 Packages
1.0.2g-1ubuntu4.6 500
500 http://security.ubuntu.com/ubuntu xenial-security/main amd64 Packages
*** 1.0.2g-1ubuntu4.5 100
100 /var/lib/dpkg/status
1.0.2g-1ubuntu4 500
500 http://us.archive.ubuntu.com/ubuntu xenial/main amd64 Packages
python-nibabel:
Installed: 2.1.0-1
Candidate: 2.1.0-1
Version table:
*** 2.1.0-1 900
900 http://http.debian.net/debian stretch/main amd64 Packages
900 http://http.debian.net/debian stretch/main i386 Packages
600 http://http.debian.net/debian sid/main amd64 Packages
600 http://http.debian.net/debian sid/main i386 Packages
100 /var/lib/dpkg/status
2.1.0-1~nd90+1 500
500 http://neuro.debian.net/debian stretch/main amd64 Packages
500 http://neuro.debian.net/debian stretch/main i386 Packages
python-biotools:
Installed: (none)
Candidate: 1.2.12-2
Version table:
1.2.12-2 600
600 http://http.debian.net/debian sid/main amd64 Packages
600 http://http.debian.net/debian sid/main i386 Packages
alienblaster:
Installed: 1.1.0-9
Candidate: 1.1.0-9
Version table:
*** 1.1.0-9 500
500 http://us.archive.ubuntu.com/ubuntu xenial/universe amd64 Packages
500 file:/my/repo ./ Packages
500 file:/my/repo2 ubuntu/ Packages
100 /var/lib/dpkg/status
skype:i386:
Installed: (none)
Candidate: (none)
Version table:
4.3.0.37-1 -1
100 /var/lib/dpkg/status
"""
out1 = {
'openssl': {
'architecture':
None,
'candidate':
'1.0.2g-1ubuntu4.8',
'installed':
'1.0.2g-1ubuntu4.5',
'versions': [{
'installed':
None,
'priority':
'500',
'sources': [{
'priority':
'500',
'source':
'http://us.archive.ubuntu.com/ubuntu '
'xenial-updates/main amd64 '
'Packages'
}],
'version':
'1.0.2g-1ubuntu4.8'
}, {
'installed':
None,
'priority':
'500',
'sources': [{
'priority':
'500',
'source':
'http://security.ubuntu.com/ubuntu '
'xenial-security/main amd64 '
'Packages'
}],
'version':
'1.0.2g-1ubuntu4.6'
}, {
'installed':
'***',
'priority':
'100',
'sources': [{
'priority': '100',
'source': '/var/lib/dpkg/status'
}],
'version':
'1.0.2g-1ubuntu4.5'
}, {
'installed':
None,
'priority':
'500',
'sources': [{
'priority':
'500',
'source':
'http://us.archive.ubuntu.com/ubuntu '
'xenial/main amd64 '
'Packages'
}],
'version':
'1.0.2g-1ubuntu4'
}]
}
}
out = parse_apt_cache_policy_pkgs_output(txt1)
assert_is_subset_recur(out1, out, [dict])
def test_conda_init_install_and_detect():
test_dir = "/tmp/niceman_conda/miniconda"
dist = CondaDistribution(
name="conda",
path=test_dir,
conda_version="4.3.31",
python_version="2.7.14.final.0",
platform=get_conda_platform_from_python(sys.platform) + "-64",
environments=[
CondaEnvironment(
name="root",
path=test_dir,
packages=[
{
"name": "conda",
"installer": None,
"version": "4.3.31",
"build": None,
"channel_name": None,
"md5": None,
"size": None,
"url": None,
"files": None,
},
{
"name": "pip",
"installer": None,
"version": "9.0.1",
"build": None,
"channel_name": None,
"md5": None,
"size": None,
"url": None,
"files": None,
},
{
"name": "pytest",
"installer": "pip",
"version": "3.4.0",
"build": None,
"channel_name": None,
"md5": None,
"size": None,
"url": None,
"files": None,
},
],
channels=[
{
"name": "conda-forge",
"url":
"https://conda.anaconda.org/conda-forge/linux-64",
},
{
"name": "defaults",
"url": "https://repo.continuum.io/pkgs/main/linux-64",
},
],
),
CondaEnvironment(
name="mytest",
path=os.path.join(test_dir, "envs/mytest"),
packages=[
{
"name": "pip",
"installer": None,
"version": "9.0.1",
"build": None,
"channel_name": None,
"md5": None,
"size": None,
"url": None,
"files": None,
},
{
"name": "xz",
"installer": None,
"version": "5.2.3",
"build": "0",
"channel_name": "conda-forge",
"md5": "f4e0d30b3caf631be7973cba1cf6f601",
"size": "874292",
"url":
"https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.3-0.tar.bz2",
"files": [
"bin/xz",
],
},
{
"name": "rpaths",
"installer": "pip",
"version": "0.13",
"build": None,
"channel_name": None,
"md5": None,
"size": None,
"url": None,
"files": [
"lib/python2.7/site-packages/rpaths.py",
],
},
],
channels=[
{
"name": "conda-forge",
"url":
"https://conda.anaconda.org/conda-forge/linux-64",
},
],
),
])
# First install the environment in /tmp/niceman_conda/miniconda
dist.initiate(None)
dist.install_packages()
# Add an empty environment to test detection of them
if not os.path.exists(os.path.join(test_dir, "envs/empty")):
call("cd " + test_dir + "; " + "./bin/conda create -y -n empty; ",
shell=True)
# Test that editable packages are detected
pymod_dir = os.path.join(test_dir, "minimal_pymodule")
if not os.path.exists(pymod_dir):
create_pymodule(pymod_dir)
call([
os.path.join(test_dir, "envs/mytest/bin/pip"), "install", "-e",
pymod_dir
])
# Now pick some files we know are in the conda install and detect them
files = [
os.path.join(test_dir, "bin/pip"),
os.path.join(test_dir, "envs/mytest/bin/xz"),
os.path.join(test_dir, "envs/empty/conda-meta/history"),
]
tracer = CondaTracer()
dists = list(tracer.identify_distributions(files))
assert len(dists) == 1, "Exactly one Conda distribution expected."
(distributions, unknown_files) = dists[0]
# NicemanProvenance.write(sys.stdout, distributions)
assert distributions.platform.startswith(
get_conda_platform_from_python(sys.platform)), \
"A conda platform is expected."
assert len(distributions.environments) == 3, \
"Three conda environments are expected."
out = {
'environments': [{
'name': 'root',
'packages': [{
'name': 'pip'
}]
}, {
'name':
'mytest',
'packages': [{
'name': 'xz'
}, {
'name': 'pip'
}, {
'name': 'rpaths',
'installer': 'pip',
'editable': False
}, {
'name': 'nmtest',
'files': [],
'installer': 'pip',
'editable': True
}]
}]
}
assert_is_subset_recur(out, attr.asdict(distributions), [dict, list])
# conda packages are not repeated as "pip" packages.
for envs in distributions.environments:
for pkg in envs.packages:
if pkg.name == "pip":
assert pkg.installer is None