def test_trace_local(trace_info): with patch("niceman.resource.ResourceManager._get_inventory") as get_inv: config = {"status": "running", "type": "shell", "name": "testing-local"} get_inv.return_value = {"testing-local": config} with patch("niceman.interface.execute.get_manager", return_value=ResourceManager()): with patch("niceman.interface.execute.CMD_CLASSES", {"trace": trace_info["class"]}): execute("ls", ["-l"], trace=True, resref="testing-local") local_dir = trace_info["local"] assert set(os.listdir(local_dir)) == {"traces", "tracers"} trace_dirs = os.listdir(op.join(local_dir, "traces")) assert len(trace_dirs) == 1 prov = NicemanProvenance(op.join(local_dir, "traces", trace_dirs[0], "niceman.yml")) deb_dists = [dist for dist in prov.get_distributions() if dist.name == "debian"] assert len(deb_dists) == 1 expect = {"packages": [{"files": ["/bin/ls"], "name": "coreutils"}]} assert_is_subset_recur(expect, attr.asdict(deb_dists[0]), [dict, list])
def __call__(path=None, spec=None, output_file=None): # heavy import -- should be delayed until actually used if not (spec or path): raise InsufficientArgumentsError( "Need at least a single --spec or a file") paths = assure_list(path) if spec: lgr.info("reading spec file %s", spec) # TODO: generic loader to auto-detect formats etc from niceman.formats.reprozip import ReprozipProvenance spec = ReprozipProvenance(spec) paths += spec.get_files() or [] # Convert paths to unicode paths = list(map(to_unicode, paths)) session = get_local_session() # TODO: at the moment assumes just a single distribution etc. # Generalize # TODO: RF so that only the above portion is reprozip specific. # If we are to reuse their layout largely -- the rest should stay as is (distributions, files) = identify_distributions(paths, session=session) from niceman.distributions.base import EnvironmentSpec spec = EnvironmentSpec(distributions=distributions, ) if files: spec.files = sorted(files) # TODO: generic writer! from niceman.formats.niceman import NicemanProvenance NicemanProvenance.write(output_file or sys.stdout, spec)
def __call__(path=None, spec=None, output_file=None, resref=None, resref_type="auto"): # heavy import -- should be delayed until actually used if not (spec or path): raise InsufficientArgumentsError( "Need at least a single --spec or a file") paths = assure_list(path) if spec: lgr.info("reading spec file %s", spec) # TODO: generic loader to auto-detect formats etc from niceman.formats.reprozip import ReprozipProvenance spec = ReprozipProvenance(spec) paths += spec.get_files() or [] # Convert paths to unicode paths = map(to_unicode, paths) # The tracers assume normalized paths. paths = list(map(normpath, paths)) if isinstance(resref, Session): # TODO: Special case for Python callers. Is this something we want # to handle more generally at the interface level? session = resref elif resref: resource = get_manager().get_resource(resref, resref_type) session = resource.get_session() else: session = get_local_session() # TODO: at the moment assumes just a single distribution etc. # Generalize # TODO: RF so that only the above portion is reprozip specific. # If we are to reuse their layout largely -- the rest should stay as is (distributions, files) = identify_distributions(paths, session=session) from niceman.distributions.base import EnvironmentSpec spec = EnvironmentSpec(distributions=distributions, ) if files: spec.files = sorted(files) # TODO: generic writer! from niceman.formats.niceman import NicemanProvenance stream = open(output_file, "w") if output_file else sys.stdout NicemanProvenance.write(stream, spec) if stream is not sys.stdout: stream.close()
def test_get_distributions(): env = NicemanProvenance(multi_debian_yaml).get_environment() with raises(ValueError): env.get_distribution(DebianDistribution) dist = env.get_distribution(CondaDistribution) assert dist is None env = NicemanProvenance(diff_1_yaml).get_environment() dist = env.get_distribution(DebianDistribution) assert isinstance(dist, DebianDistribution)
def test_conda_manager_identify_distributions(get_conda_test_dir): # Skip if network is not available (skip_if_no_network fails with fixtures) test_dir = get_conda_test_dir files = [os.path.join(test_dir, "miniconda/bin/sqlite3"), os.path.join(test_dir, "miniconda/envs/mytest/bin/xz"), os.path.join(test_dir, "miniconda/envs/mytest/lib/python2.7/site-packages/pip/index.py"), os.path.join(test_dir, "miniconda/envs/mytest/lib/python2.7/site-packages/rpaths.py"), "/sbin/iptables"] tracer = CondaTracer() dists = list(tracer.identify_distributions(files)) assert len(dists) == 1, "Exactly one Conda distribution expected." (distributions, unknown_files) = dists[0] assert unknown_files == ["/sbin/iptables"], \ "Exactly one file (/sbin/iptables) should not be discovered." assert len(distributions.environments) == 2, \ "Two conda environments are expected." out = {'environments': [{'packages': [{'files': ['bin/sqlite3'], 'name': 'sqlite'}]}, {'packages': [{'files': ['bin/xz'], 'name': 'xz'}, {'files': ['lib/python2.7/site-packages/pip/index.py'], 'name': 'pip'}, {'files': ['lib/python2.7/site-packages/rpaths.py'], 'installer': 'pip', 'name': 'rpaths'} ] } ] } assert_is_subset_recur(out, attr.asdict(distributions), [dict, list]) NicemanProvenance.write(sys.stdout, distributions) print(json.dumps(unknown_files, indent=4))
def test_write(): output = io.StringIO() # just load file_format = NicemanProvenance(NICEMAN_SPEC1_YML_FILENAME) env = file_format.get_environment() # just a basic test that we loaded stuff correctly assert len(env.distributions) == 2 assert env.distributions[0].name == 'conda' assert len(env.distributions[1].apt_sources) == 3 # and save NicemanProvenance.write(output, env) out = output.getvalue() env_reparsed = NicemanProvenance(out).get_environment() # and we could do the full round trip while retaining the same "value" assert env == env_reparsed
def test_spec_round_trip(): # FIXME: This should also test GitDistribution's, but NicemanProvenance # can't currently load those (gh-222). spec = EnvironmentSpec(distributions=[ DebianDistribution( name="debian", apt_sources=[ APTSource(name="apt_Debian_stable_main_0", component="main", archive="stable", architecture="amd64", codename="stretch", origin="Debian", label="Debian", site="ftp.us.debian.org", archive_uri="http://ftp.us.debian.org/debian", date="2018-03-10 10:21:19+00:00") ], packages=[ DEBPackage(name="debpackage"), DEBPackage(name="libc-bin", upstream_name=None, version="2.24-11+deb9u3", architecture="amd64", source_name="glibc", source_version=None, size="779468", md5="3b9aaa83b5253895b8e13509659662e4", sha1=None, sha256="aaa", versions={ "2.24-11+deb9u1": ["apt_Debian_stable_foo"], "2.24-11+deb9u3": ["apt_Debian_stable_bar", "apt__now__0"] }, install_date="2018-03-12 10:55:13+00:00", files=["/usr/bin/zdump"]) ], version="9.4"), CondaDistribution( name="conda", path="/path/to/miniconda3", conda_version="4.4.10", python_version="3.6.3.final.0", platform="linux-64", environments=[ CondaEnvironment( name="root", path="/path/to/miniconda3", packages=[ CondaPackage(name="condapkg"), CondaPackage(version="36.5.0", build="py36he42e2e1_0", name="setuptools", md5="cb1383539629db998105faf7e91e2bc7", url="https://somewhere") ], channels=[ CondaChannel(name="defaults", url="https://somewhere") ]), CondaEnvironment(name="other", path="/path/to/miniconda3", packages=[CondaPackage(name="condapkg2")]) ]), GitDistribution(name="git", packages=[GitRepo(path="/path/to/repo")]), SVNDistribution(name="svn", packages=[SVNRepo(path="/path/to/repo")]), VenvDistribution( name="venv0", path="/usr/bin/virtualenv", venv_version="15.1.0", environments=[ VenvEnvironment(path="venv-niceman", python_version="3.5.3", packages=[ VenvPackage( version="3.12", name="PyYAML", location="/path/to/venv/site-packages", local=True) ]) ]), VenvDistribution(name="venv1") ]) output = io.StringIO() NicemanProvenance.write(output, spec) loaded = NicemanProvenance(output.getvalue()).get_environment() assert spec == loaded
def __call__(prov1, prov2): env_1 = NicemanProvenance(prov1).get_environment() env_2 = NicemanProvenance(prov2).get_environment() for (dist_type, pkg_type) in ((DebianDistribution, "Debian package"), (CondaDistribution, "Conda package")): dist_1 = env_1.get_distribution(dist_type) if dist_1: pkgs_1 = { p._cmp_id: p for p in dist_1.packages } else: pkgs_1 = {} dist_2 = env_2.get_distribution(dist_type) if dist_2: pkgs_2 = { p._cmp_id: p for p in dist_2.packages } else: pkgs_2 = {} pkgs_1_s = set(pkgs_1) pkgs_2_s = set(pkgs_2) pkgs_only_1 = pkgs_1_s - pkgs_2_s pkgs_only_2 = pkgs_2_s - pkgs_1_s if pkgs_only_1 or pkgs_only_2: print(pkg_type + 's:') if pkgs_only_1: for cmp_key in sorted(pkgs_only_1): package = pkgs_1[cmp_key] print('< %s %s' % (" ".join(cmp_key), package.version)) if pkgs_only_2 and pkgs_only_2: print('---') if pkgs_only_2: for cmp_key in sorted(pkgs_only_2): package = pkgs_2[cmp_key] print('> %s %s' % (" ".join(cmp_key), package.version)) for cmp_key in pkgs_1_s.intersection(pkgs_2_s): package_1 = pkgs_1[cmp_key] package_2 = pkgs_2[cmp_key] if package_1.version != package_2.version: print('%s %s:' % (pkg_type, " ".join(cmp_key))) print('< %s' % package_1.version) print('---') print('> %s' % package_2.version) for (dist_type, repo_type) in ((GitDistribution, 'Git'), (SVNDistribution, 'SVN')): dist_1 = env_1.get_distribution(dist_type) if dist_1: repos_1 = { p.identifier: p for p in dist_1.packages } else: repos_1 = {} dist_2 = env_2.get_distribution(dist_type) if dist_2: repos_2 = { p.identifier: p for p in dist_2.packages } else: repos_2 = {} repos_1_s = set(repos_1) repos_2_s = set(repos_2) repos_1_only = repos_1_s - repos_2_s repos_2_only = repos_2_s - repos_1_s if repos_1_only or repos_2_only: print('%s repositories:' % repo_type) if repos_1_only: for repo_id in repos_1_only: repo = repos_1[repo_id] print('< %s (%s)' % (repo.identifier, repo.path)) if repos_1_only and repos_2_only: print('---') if repos_2_only: for repo_id in repos_2_only: repo = repos_2[repo_id] print('> %s (%s)' % (repo.identifier, repo.path)) for repo_id in repos_1_s.intersection(repos_2_s): repo_1 = repos_1[repo_id] repo_2 = repos_2[repo_id] if repo_1.commit == repo_2.commit: continue print('%s repository %s:' % (repo_type, repo_id)) print('< %s (%s)' % (repo_1.commit, repo_1.path)) print('---') print('> %s (%s)' % (repo_2.commit, repo_2.path)) files1 = set(env_1.files) files2 = set(env_2.files) files_1_only = files1 - files2 files_2_only = files2 - files1 if files_1_only or files_2_only: print('Files:') for fname in files_1_only: print('< %s' % fname) if files_1_only and files_2_only: print('---') for fname in files_2_only: print('> %s' % fname) return