def test_repro(self) -> None:
"""Verify that Repro object creates expected tar.gz file."""
with temporary_dir() as tmpdir:
fake_buildroot = Path(tmpdir, "buildroot")
add_file = partial(self.add_file, fake_buildroot)
add_file(".git/foo", content="foo")
add_file("dist/bar", content="bar")
add_file("baz.txt", content="baz")
add_file("qux/quux.txt", content="quux")
repro_file = Path(tmpdir, "repro.tar.gz")
repro = Repro(str(repro_file),
str(fake_buildroot),
ignore=[".git", "dist"])
repro.capture(run_info_dict={"foo": "bar", "baz": "qux"})
extract_dir = Path(tmpdir, "extract")
TGZ.extract(str(repro_file), str(extract_dir))
assert_file = partial(self.assert_file, extract_dir)
assert_file("baz.txt", expected_content="baz")
assert_file("qux/quux.txt", expected_content="quux")
assert_file("repro.sh")
assert_not_exists = partial(self.assert_not_exists, extract_dir)
assert_not_exists(".git")
assert_not_exists("dist")
def unpack_package(self, supportdir, version, filename):
tarball_filepath = self._binary_util.select_binary(
supportdir=supportdir, version=version, name=filename)
logger.debug('Tarball for %s(%s): %s', supportdir, version, tarball_filepath)
work_dir = os.path.dirname(tarball_filepath)
TGZ.extract(tarball_filepath, work_dir)
return work_dir
def test_repro(self):
"""Verify that Repro object creates expected tar.gz file"""
with temporary_dir() as tmpdir:
fake_buildroot = os.path.join(tmpdir, 'buildroot')
add_file = partial(self.add_file, fake_buildroot)
add_file('.git/foo', 'foo')
add_file('dist/bar', 'bar')
add_file('baz.txt', 'baz')
add_file('qux/quux.txt', 'quux')
repro_file = os.path.join(tmpdir, 'repro.tar.gz')
repro = Repro(repro_file, fake_buildroot, ['.git', 'dist'])
repro.capture(run_info_dict={'foo': 'bar', 'baz': 'qux'})
extract_dir = os.path.join(tmpdir, 'extract')
TGZ.extract(repro_file, extract_dir)
assert_file = partial(self.assert_file, extract_dir)
assert_file('baz.txt', 'baz')
assert_file('qux/quux.txt', 'quux')
assert_file('repro.sh')
assert_not_exists = partial(self.assert_not_exists, extract_dir)
assert_not_exists('.git')
assert_not_exists('dist')
def test_repro(self) -> None:
"""Verify that Repro object creates expected tar.gz file"""
with temporary_dir() as tmpdir:
fake_buildroot = Path(tmpdir, 'buildroot')
add_file = partial(self.add_file, fake_buildroot)
add_file('.git/foo', content='foo')
add_file('dist/bar', content='bar')
add_file('baz.txt', content='baz')
add_file('qux/quux.txt', content='quux')
repro_file = Path(tmpdir, 'repro.tar.gz')
repro = Repro(str(repro_file),
str(fake_buildroot),
ignore=['.git', 'dist'])
repro.capture(run_info_dict={'foo': 'bar', 'baz': 'qux'})
extract_dir = Path(tmpdir, 'extract')
TGZ.extract(str(repro_file), str(extract_dir))
assert_file = partial(self.assert_file, extract_dir)
assert_file('baz.txt', expected_content='baz')
assert_file('qux/quux.txt', expected_content='quux')
assert_file('repro.sh')
assert_not_exists = partial(self.assert_not_exists, extract_dir)
assert_not_exists('.git')
assert_not_exists('dist')
def test_repro(self):
"""Verify that Repro object creates expected tar.gz file"""
with temporary_dir() as tmpdir:
fake_buildroot = os.path.join(tmpdir, 'buildroot')
add_file = partial(self.add_file, fake_buildroot)
add_file('.git/foo', 'foo')
add_file('dist/bar', 'bar')
add_file('baz.txt', 'baz')
add_file('qux/quux.txt', 'quux')
repro_file = os.path.join(tmpdir, 'repro.tar.gz')
repro = Repro(repro_file, fake_buildroot, ['.git', 'dist'])
repro.capture(run_info_dict={'foo': 'bar', 'baz': 'qux'})
extract_dir = os.path.join(tmpdir, 'extract')
TGZ.extract(repro_file, extract_dir)
assert_file = partial(self.assert_file, extract_dir)
assert_file('baz.txt', 'baz')
assert_file('qux/quux.txt', 'quux')
assert_file('repro.sh')
assert_not_exists = partial(self.assert_not_exists, extract_dir)
assert_not_exists('.git')
assert_not_exists('dist')
def ensure_classfiles(target_name, classfiles):
cache_test_subdir = cache_test_subdirs[target_name]
cache_dir_entries = os.listdir(cache_test_subdir)
self.assertEqual(len(cache_dir_entries), 1)
cache_entry = cache_dir_entries[0]
with self.temporary_workdir() as cache_unzip_dir, self.temporary_workdir() as rsc_dir, self.temporary_workdir() as zinc_dir:
cache_path = os.path.join(cache_test_subdir, cache_entry)
TGZ.extract(cache_path, cache_unzip_dir)
# assert that the unzip dir has the directory structure
# ./compile/rsc/{hash}/{x}.{target_name}/{hash2}
path = descend_subdirs(cache_unzip_dir, ["compile", "rsc", None, None])
self.assertTrue(path.endswith(f".{target_name}"))
path = take_only_subdir(path)
# TODO: Surprisingly, rsc/m.jar is created even for dependee-less targets.
self.assertEqual(sorted(os.listdir(path)), ["rsc", "zinc"])
# Check that zinc/z.jar and rsc/m.jar both exist
# and that their contents contain the right classfiles
zincpath = os.path.join(path, "zinc")
zjar = os.path.join(zincpath, "z.jar")
self.assertTrue(os.path.exists(zjar))
ZIP.extract(zjar, zinc_dir)
self.assertEqual(
sorted(os.listdir(zinc_dir)), sorted(["compile_classpath", *classfiles])
)
rscpath = os.path.join(path, "rsc")
mjar = os.path.join(rscpath, "m.jar")
self.assertTrue(os.path.exists(mjar))
ZIP.extract(mjar, rsc_dir)
self.assertEqual(sorted(os.listdir(rsc_dir)), sorted(classfiles))
def unpack_package(self, supportdir, version, filename):
tarball_filepath = self._binary_util.select_binary(
supportdir=supportdir, version=version, name=filename)
logger.debug('Tarball for %s(%s): %s', supportdir, version,
tarball_filepath)
work_dir = os.path.dirname(tarball_filepath)
TGZ.extract(tarball_filepath, work_dir)
return work_dir
def get_binary_path_from_tgz(self, supportdir, version, filename, inpackage_path):
tarball_filepath = self._binary_util.select_binary(
supportdir=supportdir, version=version, name=filename)
logger.debug('Tarball for %s(%s): %s', supportdir, version, tarball_filepath)
work_dir = os.path.dirname(tarball_filepath)
unpacked_dir = os.path.join(work_dir, 'unpacked')
if not os.path.exists(unpacked_dir):
with temporary_dir(root_dir=work_dir) as tmp_dist:
TGZ.extract(tarball_filepath, tmp_dist)
os.rename(tmp_dist, unpacked_dir)
binary_path = os.path.join(unpacked_dir, inpackage_path)
return binary_path
def extracted_sdist(target, expected_prefix):
sdist = created[target]
with temporary_dir() as chroot:
TGZ.extract(sdist, chroot)
all_py_files = set()
for root, _, files in os.walk(chroot):
all_py_files.update(os.path.join(root, f) for f in files if f.endswith('.py'))
def as_full_path(p):
return os.path.join(chroot, expected_prefix, p)
yield all_py_files, as_full_path
def goroot(self):
"""Returns the $GOROOT for this go distribution.
:returns: The Go distribution $GOROOT.
:rtype: string
"""
go_distribution = self._binary_util.select_binary(self._relpath, self.version, 'go.tar.gz')
distribution_workdir = os.path.dirname(go_distribution)
outdir = os.path.join(distribution_workdir, 'unpacked')
if not os.path.exists(outdir):
with temporary_dir(root_dir=distribution_workdir) as tmp_dist:
TGZ.extract(go_distribution, tmp_dist)
os.rename(tmp_dist, outdir)
return os.path.join(outdir, 'go')
def base_dir(self):
"""Returns the base directory for this stack distribution.
:returns: The stack distribution base directory.
:rtype: string
"""
stack_dist = self._binary_util.select_binary(self._relpath, self.version, "stack.tar.gz")
distribution_workdir = os.path.dirname(stack_dist)
outdir = os.path.join(distribution_workdir, "unpacked")
if not os.path.exists(outdir):
with temporary_dir(root_dir=distribution_workdir) as tmp_dist:
TGZ.extract(stack_dist, tmp_dist)
os.rename(tmp_dist, outdir)
return os.path.join(outdir, "stack")
def path(self):
"""Returns the root path of this node distribution.
:returns: The Node distribution root path.
:rtype: string
"""
node_distribution = self._binary_util.select_binary(self._relpath, self.version, 'node.tar.gz')
distribution_workdir = os.path.dirname(node_distribution)
outdir = os.path.join(distribution_workdir, 'unpacked')
if not os.path.exists(outdir):
with temporary_dir(root_dir=distribution_workdir) as tmp_dist:
TGZ.extract(node_distribution, tmp_dist)
os.rename(tmp_dist, outdir)
return os.path.join(outdir, 'node')
def goroot(self):
"""Returns the $GOROOT for this go distribution.
:returns: The Go distribution $GOROOT.
:rtype: string
"""
go_distribution = self._binary_util.select_binary(self._relpath, self.version, 'go.tar.gz')
distribution_workdir = os.path.dirname(go_distribution)
outdir = os.path.join(distribution_workdir, 'unpacked')
if not os.path.exists(outdir):
with temporary_dir(root_dir=distribution_workdir) as tmp_dist:
TGZ.extract(go_distribution, tmp_dist)
os.rename(tmp_dist, outdir)
return os.path.join(outdir, 'go')
def path(self):
"""Returns the root path of this node distribution.
:returns: The Node distribution root path.
:rtype: string
"""
node_distribution = self._binary_util.select_binary(self._relpath, self.version, 'node.tar.gz')
distribution_workdir = os.path.dirname(node_distribution)
outdir = os.path.join(distribution_workdir, 'unpacked')
if not os.path.exists(outdir):
with temporary_dir(root_dir=distribution_workdir) as tmp_dist:
TGZ.extract(node_distribution, tmp_dist)
os.rename(tmp_dist, outdir)
return os.path.join(outdir, 'node')
def extracted_sdist(target, expected_prefix):
sdist = created[target]
with temporary_dir() as chroot:
TGZ.extract(sdist, chroot)
all_py_files = set()
for root, _, files in os.walk(chroot):
all_py_files.update(
os.path.join(root, f) for f in files
if f.endswith('.py'))
def as_full_path(p):
return os.path.join(chroot, expected_prefix, p)
yield all_py_files, as_full_path
def test_ignore_dir(self):
"""Verify that passing --repro-ignore option ignores the directory"""
# Buildroot is is based on your cwd so we need to step into a fresh
# directory for repro to look at.
root_instance = BuildRoot()
with temporary_dir() as build_root:
with root_instance.temporary(build_root):
with pushd(build_root):
with temporary_dir() as capture_dir:
add_file = partial(self.add_file, build_root)
add_file('pants.ini', '')
add_file('.git/foo', 'foo')
add_file('dist/bar', 'bar')
add_file('foo/bar', 'baz')
add_file('src/test1', 'test1')
add_file('src/test2', 'test1')
repro_file = os.path.join(capture_dir, 'repro.tar.gz')
options = {
Reproducer.options_scope:
dict(
capture=repro_file,
ignore=['src'],
)
}
repro_sub = global_subsystem_instance(Reproducer,
options=options)
repro = repro_sub.create_repro(
) # This is normally called in pants_exe.
repro.capture(run_info_dict={})
extract_loc = os.path.join(capture_dir, 'extract')
TGZ.extract(repro_file, extract_loc)
assert_file = partial(self.assert_file, extract_loc)
assert_file('foo/bar', 'baz')
assert_not_exists = partial(self.assert_not_exists,
extract_loc)
assert_not_exists('.git')
assert_not_exists('src')
def extracted_sdist(self, sdist, expected_prefix, collect_suffixes=None):
collect_suffixes = collect_suffixes or ('.py',)
def collect(path):
for suffix in collect_suffixes:
if path.endswith(suffix):
return True
return False
with temporary_dir() as chroot:
TGZ.extract(sdist, chroot)
all_py_files = set()
for root, _, files in os.walk(chroot):
all_py_files.update(os.path.join(root, f) for f in files if collect(f))
def as_full_path(p):
return os.path.join(chroot, expected_prefix, p)
yield all_py_files, as_full_path
def extracted_sdist(src, sdist, expected_prefix, collect_suffixes=None):
collect_suffixes = collect_suffixes or ('.py',)
def collect(path):
for suffix in collect_suffixes:
if path.endswith(suffix):
return True
return False
with temporary_dir() as chroot:
TGZ.extract(sdist, chroot)
all_py_files = set()
for root, _, files in os.walk(chroot):
all_py_files.update(os.path.join(root, f) for f in files if collect(f))
def as_full_path(p):
return os.path.join(chroot, expected_prefix, p)
yield all_py_files, as_full_path
def test_ignore_dir(self) -> None:
"""Verify that passing --repro-ignore option ignores the directory."""
# Buildroot is is based on your cwd so we need to step into a fresh
# directory for repro to look at.
root_instance = BuildRoot()
with temporary_dir() as build_root, root_instance.temporary(
build_root), pushd(build_root), temporary_dir() as capture_dir:
add_file = partial(self.add_file, build_root)
add_file("pants.toml")
add_file(".git/foo", content="foo")
add_file("dist/bar", content="bar")
add_file("foo/bar", content="baz")
add_file("src/test1", content="test1")
add_file("src/test2", content="test1")
repro_file = Path(capture_dir, "repro.tar.gz")
options = {
Reproducer.options_scope:
dict(
capture=str(repro_file),
ignore=["src"],
)
}
repro_sub = global_subsystem_instance(Reproducer, options=options)
repro = repro_sub.create_repro(
) # This is normally called in pants_exe.
repro.capture(run_info_dict={})
extract_loc = Path(capture_dir, "extract")
TGZ.extract(str(repro_file), str(extract_loc))
self.assert_file(extract_loc, "foo/bar", expected_content="baz")
assert_not_exists = partial(self.assert_not_exists, extract_loc)
assert_not_exists(".git")
assert_not_exists("src")
def test_ignore_dir(self):
"""Verify that passing --repro-ignore option ignores the directory"""
# Buildroot is is based on your cwd so we need to step into a fresh
# directory for repro to look at.
root_instance = BuildRoot()
with temporary_dir() as build_root:
with root_instance.temporary(build_root):
with pushd(build_root):
with temporary_dir() as capture_dir:
add_file = partial(self.add_file, build_root)
add_file('pants.ini', '')
add_file('.git/foo', 'foo')
add_file('dist/bar', 'bar')
add_file('foo/bar', 'baz')
add_file('src/test1', 'test1')
add_file('src/test2', 'test1')
repro_file = os.path.join(capture_dir, 'repro.tar.gz')
options = {
'repro': dict(
capture=repro_file,
ignore=['src'],
)}
with subsystem_instance(Reproducer, **options) as repro_sub:
repro = repro_sub.create_repro() # This is normally called in pants_exe
repro.capture(run_info_dict={})
extract_loc = os.path.join(capture_dir, 'extract')
TGZ.extract(repro_file, extract_loc)
assert_file = partial(self.assert_file, extract_loc)
assert_file('foo/bar', 'baz')
assert_not_exists = partial(self.assert_not_exists, extract_loc)
assert_not_exists('.git')
assert_not_exists('src')