def test_venv_identify_distributions(venv_test_dir): pydir = "python{v.major}.{v.minor}".format(v=sys.version_info) paths = ["lib/" + PY_VERSION + "/site-packages/yaml/parser.py", "lib/" + PY_VERSION + "/site-packages/attr/filters.py"] with chpwd(venv_test_dir): path_args = [ # Both full ... os.path.join(venv_test_dir, "venv0", paths[0]), # ... and relative paths work. os.path.join("venv1", paths[1]), ] path_args.append("/sbin/iptables") tracer = VenvTracer() dists = list(tracer.identify_distributions(path_args)) assert len(dists) == 1 distributions, unknown_files = dists[0] assert unknown_files == { "/sbin/iptables", # The editable package was added by VenvTracer as an unknown file. os.path.join(venv_test_dir, "minimal_pymodule")} assert len(distributions.environments) == 2 expect = {"environments": [{"packages": [{"files": [paths[0]], "name": "PyYAML", "editable": False}, {"files": [], "name": "nmtest", "editable": True}]}, {"packages": [{"files": [paths[1]], "name": "attrs", "editable": False}]}]} assert_is_subset_recur(expect, attr.asdict(distributions), [dict, list])
def test_venv_install(venv_test_dir, tmpdir): tmpdir = str(tmpdir) paths = [ "lib/" + PY_VERSION + "/site-packages/yaml/parser.py", "lib/" + PY_VERSION + "/site-packages/attr/filters.py" ] tracer = VenvTracer() dists = list( tracer.identify_distributions([ op.join(venv_test_dir, "venv0", paths[0]), op.join(venv_test_dir, "venv1", paths[1]) ])) assert len(dists) == 1 dist = dists[0][0] assert len(dist.environments) == 2 for idx in range(2): dist.environments[idx].path = op.join(tmpdir, "venv{}".format(idx)) dist.initiate(None) dist.install_packages() dists_installed = list( tracer.identify_distributions([ op.join(tmpdir, "venv0", paths[0]), op.join(tmpdir, "venv1", paths[1]) ])) assert len(dists_installed) == 1 dist_installed = dists_installed[0][0] expect = { "environments": [{ "packages": [{ "name": "PyYAML", "editable": False }] }, { "packages": [{ "name": "attrs", "editable": False }] }] } assert_is_subset_recur(expect, attr.asdict(dist_installed), [dict, list]) # We don't yet handle editable packages. assert any( [p.name == "nmtest" for e in dist.environments for p in e.packages]) assert not any([ p.name == "nmtest" for e in dist_installed.environments for p in e.packages ])
def test_pip_batched_show(): pkgs = ["pkg0", "pkg1", "pkg2"] batches = [ ("""\ Name: pkg0 Version: 4.1 Home-page: urlwith---three-dashes Files: file0 --- Name: pkg1 Version: 17.4.0 Files: file1""", None, None), # err, exception ("""\ Name: pkg2 Version: 4""", None, None) ] with mock.patch("niceman.distributions.piputils.execute_command_batch", return_value=batches): pkg_entries = list(piputils._pip_batched_show(None, None, pkgs)) expect = [ ( "pkg0", { "Name": "pkg0", "Version": "4.1", "Files": ["file0"], # We did not split on the URL's "---". "Home-page": "urlwith---three-dashes" }), ("pkg1", { "Name": "pkg1", "Version": "17.4.0", "Files": ["file1"] }), ("pkg2", { "Name": "pkg2", "Version": "4" }) ] assert_is_subset_recur(expect, pkg_entries, [dict, list])
def assert_distributions(result, expected_length=None, which=0, expected_unknown=None, expected_subset=None): """Wrap common assertions about identified distributions. Parameters ---------- result : iterable The result of a tracer's identify_distributions method (in its original generator form or as a list). expected_length : int, optional Expected number of items in `result`. which : int, optional Index specifying which distribution from result to consider for the `expected_unknown` and `expected_subset` assertions. expected_unknown : list, optional Which files should be marked as unknown in distribution `which`. expected_subset : dict, optional This dict is expected to be a subset of the distribution `which`. The check is done by `assert_is_subset_recur`. """ result = list(result) if expected_length is not None: assert len(result) == expected_length dist, unknown_files = result[which] if expected_unknown is not None: assert unknown_files == expected_unknown if expected_subset is not None: assert_is_subset_recur(expected_subset, attr.asdict(dist), [dict, list])
def test_conda_manager_identify_distributions(get_conda_test_dir): # Skip if network is not available (skip_if_no_network fails with fixtures) test_dir = get_conda_test_dir files = [os.path.join(test_dir, "miniconda/bin/sqlite3"), os.path.join(test_dir, "miniconda/envs/mytest/bin/xz"), os.path.join(test_dir, "miniconda/envs/mytest/lib/python2.7/site-packages/pip/index.py"), os.path.join(test_dir, "miniconda/envs/mytest/lib/python2.7/site-packages/rpaths.py"), "/sbin/iptables"] tracer = CondaTracer() dists = list(tracer.identify_distributions(files)) assert len(dists) == 1, "Exactly one Conda distribution expected." (distributions, unknown_files) = dists[0] assert unknown_files == ["/sbin/iptables"], \ "Exactly one file (/sbin/iptables) should not be discovered." assert len(distributions.environments) == 2, \ "Two conda environments are expected." out = {'environments': [{'packages': [{'files': ['bin/sqlite3'], 'name': 'sqlite'}]}, {'packages': [{'files': ['bin/xz'], 'name': 'xz'}, {'files': ['lib/python2.7/site-packages/pip/index.py'], 'name': 'pip'}, {'files': ['lib/python2.7/site-packages/rpaths.py'], 'installer': 'pip', 'name': 'rpaths'} ] } ] } assert_is_subset_recur(out, attr.asdict(distributions), [dict, list]) NicemanProvenance.write(sys.stdout, distributions) print(json.dumps(unknown_files, indent=4))
def test_parse_apt_cache_show_pkgs_output(): from ..debian import parse_apt_cache_show_pkgs_output txt1 = """\ Package: openssl Status: install ok installed Priority: optional Section: utils Installed-Size: 934 Maintainer: Ubuntu Developers <*****@*****.**> Architecture: amd64 Version: 1.0.2g-1ubuntu4.5 Depends: libc6 (>= 2.15), libssl1.0.0 (>= 1.0.2g) Suggests: ca-certificates Conffiles: /etc/ssl/openssl.cnf 7df26c55291b33344dc15e3935dabaf3 Description-en: Secure Sockets Layer toolkit - cryptographic utility This package is part of the OpenSSL project's implementation of the SSL and TLS cryptographic protocols for secure communication over the Internet. . It contains the general-purpose command line binary /usr/bin/openssl, useful for cryptographic operations such as: * creating RSA, DH, and DSA key parameters; * creating X.509 certificates, CSRs, and CRLs; * calculating message digests; * encrypting and decrypting with ciphers; * testing SSL/TLS clients and servers; * handling S/MIME signed or encrypted mail. Description-md5: 9b6de2bb6e1d9016aeb0f00bcf6617bd Original-Maintainer: Debian OpenSSL Team <*****@*****.**> Package: openssl Priority: standard Section: utils Installed-Size: 934 Maintainer: Ubuntu Developers <*****@*****.**> Original-Maintainer: Debian OpenSSL Team <*****@*****.**> Architecture: amd64 Source: openssl-src (1.0.2g) Version: 1.0.2g-1ubuntu4 Depends: libc6 (>= 2.15), libssl1.0.0 (>= 1.0.2g) Suggests: ca-certificates Filename: pool/main/o/openssl/openssl_1.0.2g-1ubuntu4_amd64.deb Size: 492190 MD5sum: 8280148dc2991da94be5810ad4d91552 SHA1: b5326f27aae83c303ff934121dede47d9fce7c76 SHA256: e897ffc8d84b0d436baca5dbd684a85146ffa78d3f2d15093779d3f5a8189690 Description-en: Secure Sockets Layer toolkit - cryptographic utility This package is part of the OpenSSL project's implementation of the SSL and TLS cryptographic protocols for secure communication over the Internet. . It contains the general-purpose command line binary /usr/bin/openssl, useful for cryptographic operations such as: * creating RSA, DH, and DSA key parameters; * creating X.509 certificates, CSRs, and CRLs; * calculating message digests; * encrypting and decrypting with ciphers; * testing SSL/TLS clients and servers; * handling S/MIME signed or encrypted mail. Description-md5: 9b6de2bb6e1d9016aeb0f00bcf6617bd Bugs: https://bugs.launchpad.net/ubuntu/+filebug Origin: Ubuntu Supported: 5y Task: standard, ubuntu-core, ubuntu-core, mythbuntu-frontend, mythbuntu-backend-slave, mythbuntu-backend-master, ubuntu-touch-core, ubuntu-touch, ubuntu-sdk-libs-tools, ubuntu-sdk Package: alienblaster Priority: extra Section: universe/games Installed-Size: 668 Maintainer: Ubuntu Developers <*****@*****.**> Original-Maintainer: Debian Games Team <*****@*****.**> Architecture: amd64 Source: alienblaster-src Version: 1.1.0-9 Depends: alienblaster-data, libc6 (>= 2.14), libgcc1 (>= 1:3.0), libsdl-mixer1.2, libsdl1.2debian (>= 1.2.11), libstdc++6 (>= 5.2) Filename: pool/universe/a/alienblaster/alienblaster_1.1.0-9_amd64.deb Size: 180278 MD5sum: e53379fd0d60e0af6304af78aa8ef2b7 SHA1: ca405056cf66a1c2ae3ae1674c22b7d24cda4986 SHA256: ff25bd843420801e9adea4f5ec1ca9656b2aeb327d8102107bf5ebbdb3046c38 Description-en: Classic 2D shoot 'em up Your mission is simple: Stop the invasion of the aliens and blast them! . Alien Blaster is a classic 2D shoot 'em up featuring lots of different weapons, special items, aliens to blast and a big bad boss. . It supports both a single player mode and a cooperative two player mode for two persons playing on one computer. Description-md5: da1f8f1a6453d62874036331e075d65f Homepage: http://www.schwardtnet.de/alienblaster/ Bugs: https://bugs.launchpad.net/ubuntu/+filebug Origin: Ubuntu """ out1 = { 'alienblaster:amd64=1.1.0-9': { 'Architecture': 'amd64', 'Source_name': 'alienblaster-src', 'Package': 'alienblaster', 'Version': '1.1.0-9' }, 'openssl:amd64=1.0.2g-1ubuntu4': { 'Architecture': 'amd64', 'Source_name': 'openssl-src', 'Source_version': '1.0.2g', 'Package': 'openssl', 'Version': '1.0.2g-1ubuntu4' }, 'openssl:amd64=1.0.2g-1ubuntu4.5': { 'Architecture': 'amd64', 'Package': 'openssl', 'Status': 'install ok installed', 'Version': '1.0.2g-1ubuntu4.5' } } out = parse_apt_cache_show_pkgs_output(txt1) assert_is_subset_recur(out1, out, [dict])
def test_parse_apt_cache_policy_source_info(): from ..debian import parse_apt_cache_policy_source_info txt = """\ Package files: 100 /var/lib/dpkg/status release a=now 500 http://neuro.debian.net/debian xenial/non-free i386 Packages release o=NeuroDebian,a=xenial,n=xenial,l=NeuroDebian,c=non-free,b=i386 origin neuro.debian.net 500 http://neuro.debian.net/debian xenial/non-free amd64 Packages release o=NeuroDebian,a=xenial,n=xenial,l=NeuroDebian,c=non-free,b=amd64 origin neuro.debian.net 500 http://neuro.debian.net/debian data/non-free i386 Packages release o=NeuroDebian,a=data,n=data,l=NeuroDebian,c=non-free,b=i386 origin neuro.debian.net 500 http://neuro.debian.net/debian data/non-free amd64 Packages release o=NeuroDebian,a=data,n=data,l=NeuroDebian,c=non-free,b=amd64 origin neuro.debian.net 500 file:/my/repo2 ubuntu/ Packages release c= 500 file:/my/repo ./ Packages release c= 500 http://dl.google.com/linux/chrome/deb stable/main amd64 Packages release v=1.0,o=Google, Inc.,a=stable,n=stable,l=Google,c=main,b=amd64 origin dl.google.com 500 http://security.ubuntu.com/ubuntu xenial-security/restricted i386 Packages release v=16.04,o=Ubuntu,a=xenial-security,n=xenial,l=Ubuntu,c=restricted,b=i386 origin security.ubuntu.com 500 http://security.ubuntu.com/ubuntu xenial-security/restricted amd64 Packages release v=16.04,o=Ubuntu,a=xenial-security,n=xenial,l=Ubuntu,c=restricted,b=amd64 origin security.ubuntu.com 500 http://debproxy:9999/debian/ jessie-backports/contrib Translation-en 100 http://debproxy:9999/debian/ jessie-backports/non-free amd64 Packages release o=Debian Backports,a=jessie-backports,n=jessie-backports,l=Debian Backports,c=non-free origin debproxy 500 http://us.archive.ubuntu.com/ubuntu xenial-updates/universe amd64 Packages release v=16.04,o=Ubuntu,a=xenial-updates,n=xenial,l=Ubuntu,c=universe,b=amd64 origin us.archive.ubuntu.com 500 http://us.archive.ubuntu.com/ubuntu xenial-updates/multiverse i386 Packages release v=16.04,o=Ubuntu,a=xenial-updates,n=xenial,l=Ubuntu,c=multiverse,b=i386 origin us.archive.ubuntu.com Pinned packages: """ out1 = { 'http://neuro.debian.net/debian xenial/non-free i386 Packages': { 'architecture': 'i386', 'archive': 'xenial', 'archive_uri': 'http://neuro.debian.net/debian', 'uri_suite': 'xenial', 'codename': 'xenial', 'component': 'non-free', 'label': 'NeuroDebian', 'origin': 'NeuroDebian', 'site': 'neuro.debian.net' }, 'http://security.ubuntu.com/ubuntu xenial-security/restricted amd64 Packages': { 'architecture': 'amd64', 'archive': 'xenial-security', 'archive_uri': 'http://security.ubuntu.com/ubuntu', 'uri_suite': 'xenial-security', 'codename': 'xenial', 'component': 'restricted', 'label': 'Ubuntu', 'origin': 'Ubuntu', 'site': 'security.ubuntu.com' }, 'http://debproxy:9999/debian/ jessie-backports/contrib Translation-en': { 'archive_uri': 'http://debproxy:9999/debian/', 'uri_suite': 'jessie-backports' }, 'http://debproxy:9999/debian/ jessie-backports/non-free amd64 Packages': { 'archive': 'jessie-backports', 'archive_uri': 'http://debproxy:9999/debian/', 'codename': 'jessie-backports', 'component': 'non-free', 'label': 'Debian Backports', 'origin': 'Debian Backports', 'site': 'debproxy', 'uri_suite': 'jessie-backports' }, } out = parse_apt_cache_policy_source_info(txt) assert_is_subset_recur(out1, out, [dict])
def test_parse_apt_cache_policy_pkgs_output(): from ..debian import parse_apt_cache_policy_pkgs_output txt1 = """\ afni: Installed: 16.2.07~dfsg.1-2~nd90+1 Candidate: 16.2.07~dfsg.1-2~nd90+1 Version table: *** 16.2.07~dfsg.1-2~nd90+1 500 500 http://neuro.debian.net/debian stretch/contrib amd64 Packages 100 /var/lib/dpkg/status openssl: Installed: 1.0.2g-1ubuntu4.5 Candidate: 1.0.2g-1ubuntu4.8 Version table: 1.0.2g-1ubuntu4.8 500 500 http://us.archive.ubuntu.com/ubuntu xenial-updates/main amd64 Packages 1.0.2g-1ubuntu4.6 500 500 http://security.ubuntu.com/ubuntu xenial-security/main amd64 Packages *** 1.0.2g-1ubuntu4.5 100 100 /var/lib/dpkg/status 1.0.2g-1ubuntu4 500 500 http://us.archive.ubuntu.com/ubuntu xenial/main amd64 Packages python-nibabel: Installed: 2.1.0-1 Candidate: 2.1.0-1 Version table: *** 2.1.0-1 900 900 http://http.debian.net/debian stretch/main amd64 Packages 900 http://http.debian.net/debian stretch/main i386 Packages 600 http://http.debian.net/debian sid/main amd64 Packages 600 http://http.debian.net/debian sid/main i386 Packages 100 /var/lib/dpkg/status 2.1.0-1~nd90+1 500 500 http://neuro.debian.net/debian stretch/main amd64 Packages 500 http://neuro.debian.net/debian stretch/main i386 Packages python-biotools: Installed: (none) Candidate: 1.2.12-2 Version table: 1.2.12-2 600 600 http://http.debian.net/debian sid/main amd64 Packages 600 http://http.debian.net/debian sid/main i386 Packages alienblaster: Installed: 1.1.0-9 Candidate: 1.1.0-9 Version table: *** 1.1.0-9 500 500 http://us.archive.ubuntu.com/ubuntu xenial/universe amd64 Packages 500 file:/my/repo ./ Packages 500 file:/my/repo2 ubuntu/ Packages 100 /var/lib/dpkg/status skype:i386: Installed: (none) Candidate: (none) Version table: 4.3.0.37-1 -1 100 /var/lib/dpkg/status """ out1 = { 'openssl': { 'architecture': None, 'candidate': '1.0.2g-1ubuntu4.8', 'installed': '1.0.2g-1ubuntu4.5', 'versions': [{ 'installed': None, 'priority': '500', 'sources': [{ 'priority': '500', 'source': 'http://us.archive.ubuntu.com/ubuntu ' 'xenial-updates/main amd64 ' 'Packages' }], 'version': '1.0.2g-1ubuntu4.8' }, { 'installed': None, 'priority': '500', 'sources': [{ 'priority': '500', 'source': 'http://security.ubuntu.com/ubuntu ' 'xenial-security/main amd64 ' 'Packages' }], 'version': '1.0.2g-1ubuntu4.6' }, { 'installed': '***', 'priority': '100', 'sources': [{ 'priority': '100', 'source': '/var/lib/dpkg/status' }], 'version': '1.0.2g-1ubuntu4.5' }, { 'installed': None, 'priority': '500', 'sources': [{ 'priority': '500', 'source': 'http://us.archive.ubuntu.com/ubuntu ' 'xenial/main amd64 ' 'Packages' }], 'version': '1.0.2g-1ubuntu4' }] } } out = parse_apt_cache_policy_pkgs_output(txt1) assert_is_subset_recur(out1, out, [dict])
def test_conda_init_install_and_detect(): test_dir = "/tmp/niceman_conda/miniconda" dist = CondaDistribution( name="conda", path=test_dir, conda_version="4.3.31", python_version="2.7.14.final.0", platform=get_conda_platform_from_python(sys.platform) + "-64", environments=[ CondaEnvironment( name="root", path=test_dir, packages=[ { "name": "conda", "installer": None, "version": "4.3.31", "build": None, "channel_name": None, "md5": None, "size": None, "url": None, "files": None, }, { "name": "pip", "installer": None, "version": "9.0.1", "build": None, "channel_name": None, "md5": None, "size": None, "url": None, "files": None, }, { "name": "pytest", "installer": "pip", "version": "3.4.0", "build": None, "channel_name": None, "md5": None, "size": None, "url": None, "files": None, }, ], channels=[ { "name": "conda-forge", "url": "https://conda.anaconda.org/conda-forge/linux-64", }, { "name": "defaults", "url": "https://repo.continuum.io/pkgs/main/linux-64", }, ], ), CondaEnvironment( name="mytest", path=os.path.join(test_dir, "envs/mytest"), packages=[ { "name": "pip", "installer": None, "version": "9.0.1", "build": None, "channel_name": None, "md5": None, "size": None, "url": None, "files": None, }, { "name": "xz", "installer": None, "version": "5.2.3", "build": "0", "channel_name": "conda-forge", "md5": "f4e0d30b3caf631be7973cba1cf6f601", "size": "874292", "url": "https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.3-0.tar.bz2", "files": [ "bin/xz", ], }, { "name": "rpaths", "installer": "pip", "version": "0.13", "build": None, "channel_name": None, "md5": None, "size": None, "url": None, "files": [ "lib/python2.7/site-packages/rpaths.py", ], }, ], channels=[ { "name": "conda-forge", "url": "https://conda.anaconda.org/conda-forge/linux-64", }, ], ), ]) # First install the environment in /tmp/niceman_conda/miniconda dist.initiate(None) dist.install_packages() # Add an empty environment to test detection of them if not os.path.exists(os.path.join(test_dir, "envs/empty")): call("cd " + test_dir + "; " + "./bin/conda create -y -n empty; ", shell=True) # Test that editable packages are detected pymod_dir = os.path.join(test_dir, "minimal_pymodule") if not os.path.exists(pymod_dir): create_pymodule(pymod_dir) call([ os.path.join(test_dir, "envs/mytest/bin/pip"), "install", "-e", pymod_dir ]) # Now pick some files we know are in the conda install and detect them files = [ os.path.join(test_dir, "bin/pip"), os.path.join(test_dir, "envs/mytest/bin/xz"), os.path.join(test_dir, "envs/empty/conda-meta/history"), ] tracer = CondaTracer() dists = list(tracer.identify_distributions(files)) assert len(dists) == 1, "Exactly one Conda distribution expected." (distributions, unknown_files) = dists[0] # NicemanProvenance.write(sys.stdout, distributions) assert distributions.platform.startswith( get_conda_platform_from_python(sys.platform)), \ "A conda platform is expected." assert len(distributions.environments) == 3, \ "Three conda environments are expected." out = { 'environments': [{ 'name': 'root', 'packages': [{ 'name': 'pip' }] }, { 'name': 'mytest', 'packages': [{ 'name': 'xz' }, { 'name': 'pip' }, { 'name': 'rpaths', 'installer': 'pip', 'editable': False }, { 'name': 'nmtest', 'files': [], 'installer': 'pip', 'editable': True }] }] } assert_is_subset_recur(out, attr.asdict(distributions), [dict, list]) # conda packages are not repeated as "pip" packages. for envs in distributions.environments: for pkg in envs.packages: if pkg.name == "pip": assert pkg.installer is None