def test_trace_local(trace_info):
with patch("niceman.resource.ResourceManager._get_inventory") as get_inv:
config = {"status": "running",
"type": "shell",
"name": "testing-local"}
get_inv.return_value = {"testing-local": config}
with patch("niceman.interface.execute.get_manager",
return_value=ResourceManager()):
with patch("niceman.interface.execute.CMD_CLASSES",
{"trace": trace_info["class"]}):
execute("ls", ["-l"], trace=True, resref="testing-local")
local_dir = trace_info["local"]
assert set(os.listdir(local_dir)) == {"traces", "tracers"}
trace_dirs = os.listdir(op.join(local_dir, "traces"))
assert len(trace_dirs) == 1
prov = NicemanProvenance(op.join(local_dir, "traces",
trace_dirs[0], "niceman.yml"))
deb_dists = [dist for dist in prov.get_distributions()
if dist.name == "debian"]
assert len(deb_dists) == 1
expect = {"packages": [{"files": ["/bin/ls"], "name": "coreutils"}]}
assert_is_subset_recur(expect, attr.asdict(deb_dists[0]), [dict, list])
def __call__(path=None, spec=None, output_file=None):
# heavy import -- should be delayed until actually used
if not (spec or path):
raise InsufficientArgumentsError(
"Need at least a single --spec or a file")
paths = assure_list(path)
if spec:
lgr.info("reading spec file %s", spec)
# TODO: generic loader to auto-detect formats etc
from niceman.formats.reprozip import ReprozipProvenance
spec = ReprozipProvenance(spec)
paths += spec.get_files() or []
# Convert paths to unicode
paths = list(map(to_unicode, paths))
session = get_local_session()
# TODO: at the moment assumes just a single distribution etc.
# Generalize
# TODO: RF so that only the above portion is reprozip specific.
# If we are to reuse their layout largely -- the rest should stay as is
(distributions, files) = identify_distributions(paths, session=session)
from niceman.distributions.base import EnvironmentSpec
spec = EnvironmentSpec(distributions=distributions, )
if files:
spec.files = sorted(files)
# TODO: generic writer!
from niceman.formats.niceman import NicemanProvenance
NicemanProvenance.write(output_file or sys.stdout, spec)
def __call__(path=None,
spec=None,
output_file=None,
resref=None,
resref_type="auto"):
# heavy import -- should be delayed until actually used
if not (spec or path):
raise InsufficientArgumentsError(
"Need at least a single --spec or a file")
paths = assure_list(path)
if spec:
lgr.info("reading spec file %s", spec)
# TODO: generic loader to auto-detect formats etc
from niceman.formats.reprozip import ReprozipProvenance
spec = ReprozipProvenance(spec)
paths += spec.get_files() or []
# Convert paths to unicode
paths = map(to_unicode, paths)
# The tracers assume normalized paths.
paths = list(map(normpath, paths))
if isinstance(resref, Session):
# TODO: Special case for Python callers. Is this something we want
# to handle more generally at the interface level?
session = resref
elif resref:
resource = get_manager().get_resource(resref, resref_type)
session = resource.get_session()
else:
session = get_local_session()
# TODO: at the moment assumes just a single distribution etc.
# Generalize
# TODO: RF so that only the above portion is reprozip specific.
# If we are to reuse their layout largely -- the rest should stay as is
(distributions, files) = identify_distributions(paths, session=session)
from niceman.distributions.base import EnvironmentSpec
spec = EnvironmentSpec(distributions=distributions, )
if files:
spec.files = sorted(files)
# TODO: generic writer!
from niceman.formats.niceman import NicemanProvenance
stream = open(output_file, "w") if output_file else sys.stdout
NicemanProvenance.write(stream, spec)
if stream is not sys.stdout:
stream.close()
def test_get_distributions():
env = NicemanProvenance(multi_debian_yaml).get_environment()
with raises(ValueError):
env.get_distribution(DebianDistribution)
dist = env.get_distribution(CondaDistribution)
assert dist is None
env = NicemanProvenance(diff_1_yaml).get_environment()
dist = env.get_distribution(DebianDistribution)
assert isinstance(dist, DebianDistribution)
def test_conda_manager_identify_distributions(get_conda_test_dir):
# Skip if network is not available (skip_if_no_network fails with fixtures)
test_dir = get_conda_test_dir
files = [os.path.join(test_dir, "miniconda/bin/sqlite3"),
os.path.join(test_dir, "miniconda/envs/mytest/bin/xz"),
os.path.join(test_dir, "miniconda/envs/mytest/lib/python2.7/site-packages/pip/index.py"),
os.path.join(test_dir, "miniconda/envs/mytest/lib/python2.7/site-packages/rpaths.py"),
"/sbin/iptables"]
tracer = CondaTracer()
dists = list(tracer.identify_distributions(files))
assert len(dists) == 1, "Exactly one Conda distribution expected."
(distributions, unknown_files) = dists[0]
assert unknown_files == ["/sbin/iptables"], \
"Exactly one file (/sbin/iptables) should not be discovered."
assert len(distributions.environments) == 2, \
"Two conda environments are expected."
out = {'environments': [{'packages': [{'files': ['bin/sqlite3'],
'name': 'sqlite'}]},
{'packages': [{'files': ['bin/xz'],
'name': 'xz'},
{'files': ['lib/python2.7/site-packages/pip/index.py'],
'name': 'pip'},
{'files': ['lib/python2.7/site-packages/rpaths.py'],
'installer': 'pip',
'name': 'rpaths'}
]
}
]
}
assert_is_subset_recur(out, attr.asdict(distributions), [dict, list])
NicemanProvenance.write(sys.stdout, distributions)
print(json.dumps(unknown_files, indent=4))
def test_write():
output = io.StringIO()
# just load
file_format = NicemanProvenance(NICEMAN_SPEC1_YML_FILENAME)
env = file_format.get_environment()
# just a basic test that we loaded stuff correctly
assert len(env.distributions) == 2
assert env.distributions[0].name == 'conda'
assert len(env.distributions[1].apt_sources) == 3
# and save
NicemanProvenance.write(output, env)
out = output.getvalue()
env_reparsed = NicemanProvenance(out).get_environment()
# and we could do the full round trip while retaining the same "value"
assert env == env_reparsed
def test_spec_round_trip():
# FIXME: This should also test GitDistribution's, but NicemanProvenance
# can't currently load those (gh-222).
spec = EnvironmentSpec(distributions=[
DebianDistribution(
name="debian",
apt_sources=[
APTSource(name="apt_Debian_stable_main_0",
component="main",
archive="stable",
architecture="amd64",
codename="stretch",
origin="Debian",
label="Debian",
site="ftp.us.debian.org",
archive_uri="http://ftp.us.debian.org/debian",
date="2018-03-10 10:21:19+00:00")
],
packages=[
DEBPackage(name="debpackage"),
DEBPackage(name="libc-bin",
upstream_name=None,
version="2.24-11+deb9u3",
architecture="amd64",
source_name="glibc",
source_version=None,
size="779468",
md5="3b9aaa83b5253895b8e13509659662e4",
sha1=None,
sha256="aaa",
versions={
"2.24-11+deb9u1": ["apt_Debian_stable_foo"],
"2.24-11+deb9u3":
["apt_Debian_stable_bar", "apt__now__0"]
},
install_date="2018-03-12 10:55:13+00:00",
files=["/usr/bin/zdump"])
],
version="9.4"),
CondaDistribution(
name="conda",
path="/path/to/miniconda3",
conda_version="4.4.10",
python_version="3.6.3.final.0",
platform="linux-64",
environments=[
CondaEnvironment(
name="root",
path="/path/to/miniconda3",
packages=[
CondaPackage(name="condapkg"),
CondaPackage(version="36.5.0",
build="py36he42e2e1_0",
name="setuptools",
md5="cb1383539629db998105faf7e91e2bc7",
url="https://somewhere")
],
channels=[
CondaChannel(name="defaults", url="https://somewhere")
]),
CondaEnvironment(name="other",
path="/path/to/miniconda3",
packages=[CondaPackage(name="condapkg2")])
]),
GitDistribution(name="git", packages=[GitRepo(path="/path/to/repo")]),
SVNDistribution(name="svn", packages=[SVNRepo(path="/path/to/repo")]),
VenvDistribution(
name="venv0",
path="/usr/bin/virtualenv",
venv_version="15.1.0",
environments=[
VenvEnvironment(path="venv-niceman",
python_version="3.5.3",
packages=[
VenvPackage(
version="3.12",
name="PyYAML",
location="/path/to/venv/site-packages",
local=True)
])
]),
VenvDistribution(name="venv1")
])
output = io.StringIO()
NicemanProvenance.write(output, spec)
loaded = NicemanProvenance(output.getvalue()).get_environment()
assert spec == loaded
def __call__(prov1, prov2):
env_1 = NicemanProvenance(prov1).get_environment()
env_2 = NicemanProvenance(prov2).get_environment()
for (dist_type, pkg_type) in ((DebianDistribution, "Debian package"),
(CondaDistribution, "Conda package")):
dist_1 = env_1.get_distribution(dist_type)
if dist_1:
pkgs_1 = { p._cmp_id: p for p in dist_1.packages }
else:
pkgs_1 = {}
dist_2 = env_2.get_distribution(dist_type)
if dist_2:
pkgs_2 = { p._cmp_id: p for p in dist_2.packages }
else:
pkgs_2 = {}
pkgs_1_s = set(pkgs_1)
pkgs_2_s = set(pkgs_2)
pkgs_only_1 = pkgs_1_s - pkgs_2_s
pkgs_only_2 = pkgs_2_s - pkgs_1_s
if pkgs_only_1 or pkgs_only_2:
print(pkg_type + 's:')
if pkgs_only_1:
for cmp_key in sorted(pkgs_only_1):
package = pkgs_1[cmp_key]
print('< %s %s' % (" ".join(cmp_key), package.version))
if pkgs_only_2 and pkgs_only_2:
print('---')
if pkgs_only_2:
for cmp_key in sorted(pkgs_only_2):
package = pkgs_2[cmp_key]
print('> %s %s' % (" ".join(cmp_key), package.version))
for cmp_key in pkgs_1_s.intersection(pkgs_2_s):
package_1 = pkgs_1[cmp_key]
package_2 = pkgs_2[cmp_key]
if package_1.version != package_2.version:
print('%s %s:' % (pkg_type, " ".join(cmp_key)))
print('< %s' % package_1.version)
print('---')
print('> %s' % package_2.version)
for (dist_type, repo_type) in ((GitDistribution, 'Git'),
(SVNDistribution, 'SVN')):
dist_1 = env_1.get_distribution(dist_type)
if dist_1:
repos_1 = { p.identifier: p for p in dist_1.packages }
else:
repos_1 = {}
dist_2 = env_2.get_distribution(dist_type)
if dist_2:
repos_2 = { p.identifier: p for p in dist_2.packages }
else:
repos_2 = {}
repos_1_s = set(repos_1)
repos_2_s = set(repos_2)
repos_1_only = repos_1_s - repos_2_s
repos_2_only = repos_2_s - repos_1_s
if repos_1_only or repos_2_only:
print('%s repositories:' % repo_type)
if repos_1_only:
for repo_id in repos_1_only:
repo = repos_1[repo_id]
print('< %s (%s)' % (repo.identifier, repo.path))
if repos_1_only and repos_2_only:
print('---')
if repos_2_only:
for repo_id in repos_2_only:
repo = repos_2[repo_id]
print('> %s (%s)' % (repo.identifier, repo.path))
for repo_id in repos_1_s.intersection(repos_2_s):
repo_1 = repos_1[repo_id]
repo_2 = repos_2[repo_id]
if repo_1.commit == repo_2.commit:
continue
print('%s repository %s:' % (repo_type, repo_id))
print('< %s (%s)' % (repo_1.commit, repo_1.path))
print('---')
print('> %s (%s)' % (repo_2.commit, repo_2.path))
files1 = set(env_1.files)
files2 = set(env_2.files)
files_1_only = files1 - files2
files_2_only = files2 - files1
if files_1_only or files_2_only:
print('Files:')
for fname in files_1_only:
print('< %s' % fname)
if files_1_only and files_2_only:
print('---')
for fname in files_2_only:
print('> %s' % fname)
return