def test_changed_with_multiple_build_files(self):
new_build_file = 'src/python/python_targets/BUILD.new'
with create_isolated_git_repo() as worktree:
touch(os.path.join(worktree, new_build_file))
stdout_data = self.run_list([])
self.assertEqual(stdout_data.strip(), '')
def test_junit_run_target_cwd_trumps_cwd_option(self):
with temporary_dir() as target_cwd:
self.make_target(
spec='tests/java/org/pantsbuild/foo:foo_test',
target_type=JUnitTests,
sources=['FooTest.java'],
cwd=target_cwd
)
content = dedent("""
package org.pantsbuild.foo;
import java.io.File;
import org.junit.Test;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
public class FooTest {
@Test
public void testFoo() {
assertTrue(new File("target_cwd_sentinel").exists());
assertFalse(new File("option_cwd_sentinel").exists());
}
}
""")
touch(os.path.join(target_cwd, 'target_cwd_sentinel'))
with temporary_dir() as option_cwd:
touch(os.path.join(option_cwd, 'option_cwd_sentinel'))
self.set_options(cwd=option_cwd)
self._execute_junit_runner([('FooTest.java', content)],
target_name='tests/java/org/pantsbuild/foo:foo_test')
def plugin_resolution(chroot=None, plugins=None):
@contextmanager
def provide_chroot(existing):
if existing:
yield existing, False
else:
with temporary_dir() as new_chroot:
yield new_chroot, True
with provide_chroot(chroot) as (root_dir, create_artifacts):
env = {'PANTS_BOOTSTRAPDIR': root_dir}
repo_dir = None
if plugins:
repo_dir = os.path.join(root_dir, 'repo')
env.update(PANTS_PYTHON_REPOS_REPOS='[{!r}]'.format(repo_dir),
PANTS_PYTHON_REPOS_INDEXES='[]',
PANTS_PYTHON_SETUP_RESOLVER_CACHE_TTL='1')
plugin_list = []
for plugin in plugins:
version = None
if isinstance(plugin, tuple):
plugin, version = plugin
plugin_list.append('{}=={}'.format(plugin, version) if version else plugin)
if create_artifacts:
create_plugin(repo_dir, plugin, version)
env['PANTS_PLUGINS'] = '[{}]'.format(','.join(map(repr, plugin_list)))
configpath = os.path.join(root_dir, 'pants.ini')
if create_artifacts:
touch(configpath)
options_bootstrapper = OptionsBootstrapper(env=env, configpath=configpath, args=[])
plugin_resolver = PluginResolver(options_bootstrapper)
cache_dir = plugin_resolver.plugin_cache_dir
yield plugin_resolver.resolve(WorkingSet(entries=[])), root_dir, repo_dir, cache_dir
def test_relative_symlink_overwrite_existing_file(self):
# Succeeds, since os.unlink can be safely called on files that aren't symlinks.
with temporary_dir() as tmpdir_1: # source and link in same dir
source = os.path.join(tmpdir_1, 'source')
link_path = os.path.join(tmpdir_1, 'link')
touch(link_path)
relative_symlink(source, link_path)
def test_node_deployable_bundle(self):
with temporary_dir() as tmp_dir:
touch(os.path.join(tmp_dir, 'a'))
node_module_target = self.make_target(self.node_module_target_name_full, NodeModule)
target = self.make_target(
self.target_name_full, NodeBundle,
node_module=self.node_module_target_name_full,
dependencies=[node_module_target])
bundleable_js_product = defaultdict(MultipleRootedProducts)
bundleable_js_product[node_module_target].add_abs_paths(tmp_dir, [tmp_dir])
task_context = self.context(target_roots=[target])
task_context.products.safe_create_data(
'bundleable_js', init_func=lambda: bundleable_js_product)
task = self.create_task(task_context)
task.execute()
product = task_context.products.get('deployable_archives')
self.assertIsNotNone(product)
self.assertFalse(product.empty())
product_data = product.get(target)
self.assertIsNotNone(product_data)
product_basedir = list(product_data.keys())[0]
self.assertEqual(product_data[product_basedir], ['{}.tar.gz'.format(self.target_name)])
def _bootstrap_ivy(self, bootstrap_jar_path):
options = self._ivy_subsystem.get_options()
if not os.path.exists(bootstrap_jar_path):
with temporary_file() as bootstrap_jar:
fetcher = Fetcher()
checksummer = fetcher.ChecksumListener(digest=hashlib.sha1())
try:
logger.info('\nDownloading {}'.format(options.bootstrap_jar_url))
# TODO: Capture the stdout of the fetcher, instead of letting it output
# to the console directly.
fetcher.download(options.bootstrap_jar_url,
listener=fetcher.ProgressListener().wrap(checksummer),
path_or_fd=bootstrap_jar,
timeout_secs=options.bootstrap_fetch_timeout_secs)
logger.info('sha1: {}'.format(checksummer.checksum))
bootstrap_jar.close()
touch(bootstrap_jar_path)
shutil.move(bootstrap_jar.name, bootstrap_jar_path)
except fetcher.Error as e:
raise self.Error('Problem fetching the ivy bootstrap jar! {}'.format(e))
return Ivy(bootstrap_jar_path,
ivy_settings=options.bootstrap_ivy_settings or options.ivy_settings,
ivy_cache_dir=options.cache_dir,
extra_jvm_options=self._ivy_subsystem.extra_jvm_options())
def test_rm_rf_permission_error_raises(self, file_name='./perm_guarded_file'):
with temporary_dir() as td, pushd(td), \
mock.patch('pants.util.dirutil.shutil.rmtree') as mock_rmtree, \
self.assertRaises(OSError):
mock_rmtree.side_effect = OSError(errno.EACCES, os.strerror(errno.EACCES))
touch(file_name)
rm_rf(file_name)
def test_pantsd_invalidation_file_tracking(self):
test_file = 'testprojects/src/python/print_env/main.py'
config = {'GLOBAL': {'pantsd_invalidation_globs': '["testprojects/src/python/print_env/*"]'}}
with self.pantsd_successful_run_context(extra_config=config) as (
pantsd_run, checker, workdir, _
):
pantsd_run(['help'])
checker.assert_started()
# Let any fs events quiesce.
time.sleep(5)
# Check the logs.
self.assertRegexpMatches(
full_pantsd_log(workdir),
r'watching invalidating files:.*{}'.format(test_file)
)
checker.assert_running()
touch(test_file)
# Permit ample time for the async file event propagate in CI.
time.sleep(10)
checker.assert_stopped()
self.assertIn('saw file events covered by invalidation globs', full_pantsd_log(workdir))
def setUp(self):
self.base_dir = tempfile.mkdtemp()
self._project_tree = None
# Seed a BUILD outside the build root that should not be detected
touch(os.path.join(self.base_dir, 'BUILD'))
self.root_dir = os.path.join(self.base_dir, 'root')
self.touch('grandparent/parent/BUILD')
self.touch('grandparent/parent/BUILD.twitter')
# Tricky! This is a directory
self.makedirs('grandparent/parent/BUILD.dir')
self.makedirs('grandparent/BUILD')
self.touch('BUILD')
self.touch('BUILD.twitter')
self.touch('grandparent/parent/child1/BUILD')
self.touch('grandparent/parent/child1/BUILD.twitter')
self.touch('grandparent/parent/child2/child3/BUILD')
self.makedirs('grandparent/parent/child2/BUILD')
self.makedirs('grandparent/parent/child4')
self.touch('grandparent/parent/child5/BUILD')
self.makedirs('path-that-does-exist')
self.touch('path-that-does-exist/BUILD.invalid.suffix')
# This exercises https://github.com/pantsbuild/pants/issues/1742
# Prior to that fix, BUILD directories were handled, but not if there was a valid BUILD file
# sibling.
self.makedirs('issue_1742/BUILD')
self.touch('issue_1742/BUILD.sibling')
def workspace(self, *buildfiles):
with temporary_dir() as root_dir:
with BuildRoot().temporary(root_dir):
with pushd(root_dir):
for buildfile in buildfiles:
touch(os.path.join(root_dir, buildfile))
yield os.path.realpath(root_dir)
def setUp(self):
self.origin = safe_mkdtemp()
with pushd(self.origin):
subprocess.check_call(['git', 'init', '--bare'])
self.gitdir = safe_mkdtemp()
self.worktree = safe_mkdtemp()
self.readme_file = os.path.join(self.worktree, 'README')
with environment_as(GIT_DIR=self.gitdir, GIT_WORK_TREE=self.worktree):
self.init_repo('depot', self.origin)
touch(self.readme_file)
subprocess.check_call(['git', 'add', 'README'])
subprocess.check_call(['git', 'commit', '-am', 'initial commit with decode -> \x81b'])
subprocess.check_call(['git', 'tag', 'first'])
subprocess.check_call(['git', 'push', '--tags', 'depot', 'master'])
subprocess.check_call(['git', 'branch', '--set-upstream', 'master', 'depot/master'])
with safe_open(self.readme_file, 'w') as readme:
readme.write('Hello World.')
subprocess.check_call(['git', 'commit', '-am', 'Update README.'])
self.clone2 = safe_mkdtemp()
with pushd(self.clone2):
self.init_repo('origin', self.origin)
subprocess.check_call(['git', 'pull', '--tags', 'origin', 'master:master'])
with safe_open(os.path.realpath('README'), 'a') as readme:
readme.write('--')
subprocess.check_call(['git', 'commit', '-am', 'Update README 2.'])
subprocess.check_call(['git', 'push', '--tags', 'origin', 'master'])
self.git = Git(gitdir=self.gitdir, worktree=self.worktree)
def test_leave_one(self):
"""Ensure that max-old of 1 removes all but one files"""
with temporary_dir() as cache_dir:
artifact_dir = os.path.join(cache_dir, ZincCompile.stable_name(),
'testprojects.src.java.org.pantsbuild.testproject.unicode.main.main')
touch(os.path.join(artifact_dir, 'old_cache_test1'))
touch(os.path.join(artifact_dir, 'old_cache_test2'))
touch(os.path.join(artifact_dir, 'old_cache_test3'))
touch(os.path.join(artifact_dir, 'old_cache_test4'))
touch(os.path.join(artifact_dir, 'old_cache_test5'))
config = {'cache.compile.zinc': {'write_to': [cache_dir]}}
pants_run = self.run_pants(self.create_platform_args(6) +
['compile.zinc',
'testprojects/src/java/org/pantsbuild/testproject/unicode/main',
'--cache-max-entries-per-target=1'],
config=config)
self.assert_success(pants_run)
# One artifact for java 6
self.assertEqual(len(os.listdir(artifact_dir)), 1)
# Rerun for java 7
pants_run = self.run_pants(self.create_platform_args(7) +
['compile.zinc',
'testprojects/src/java/org/pantsbuild/testproject/unicode/main',
'--cache-max-entries-per-target=1'],
config)
self.assert_success(pants_run)
# One artifact for java 7
self.assertEqual(len(os.listdir(artifact_dir)), 1)
def test_bundled_classpath(self):
"""This creates the following classpath
basedir/libs/A.jar:basedir/resources
"""
RESOURCES = 'resources'
LIB_DIR = 'libs'
JAR_FILE = 'A.jar'
basedir = safe_mkdtemp()
lib_dir = os.path.join(basedir, LIB_DIR)
resource_dir = os.path.join(lib_dir, RESOURCES)
jar_file = os.path.join(lib_dir, JAR_FILE)
for dir in (lib_dir, resource_dir):
safe_mkdir(dir)
touch(jar_file)
classpath = [jar_file, resource_dir]
with bundled_classpath(classpath) as bundled_cp:
self.assertEquals(1, len(bundled_cp))
bundled_jar = bundled_cp[0]
self.assertTrue(os.path.exists(bundled_jar))
with open_zip(bundled_jar) as synthetic_jar:
self.assertListEqual([Manifest.PATH], synthetic_jar.namelist())
# manifest should contain the absolute path of both jar and resource directory
self.assertEquals('{}: {} {}/\n'.format(Manifest.CLASS_PATH, os.path.realpath(jar_file),
os.path.realpath(resource_dir)),
synthetic_jar.read(Manifest.PATH).replace('\n ', ''))
safe_rmtree(resource_dir)
def _create_package_structure(self, workdir, module):
path = workdir
for d in module.split('.'):
path = os.path.join(path, d)
# Supposedly we get handed a clean workdir, but I'm not sure that's true.
safe_mkdir(path, clean=True)
touch(os.path.join(path, '__init__.py'))
return path
def tool(self, name):
with temporary_dir() as tool_root:
tool_path = os.path.join(tool_root, name)
touch(tool_path)
chmod_plus_x(tool_path)
new_path = os.pathsep.join([tool_root] + os.environ.get('PATH', '').split(os.pathsep))
with environment_as(PATH=new_path):
yield tool_path
def test_file_no_perms(self):
with temporary_dir() as base:
no_perms = os.path.join(base, 'dne')
touch(no_perms)
os.chmod(no_perms, 0)
self.assertTrue(os.path.exists(no_perms))
with self.assertRaises(self.fetcher.PermanentError):
self.fetcher.fetch(no_perms, self.listener)
def test_when_key_has_associated_directory_cleans_dir(self):
with temporary_dir() as tmpdir:
filename = os.path.join(tmpdir, 'deleted')
touch(filename)
key = 'some-key'
cache_key_to_class_dir = {key: tmpdir}
CacheHitCallback(cache_key_to_class_dir)(key)
self.assertFalse(os.path.exists(filename))
def test_changed_with_multiple_build_files(self):
new_build_file = 'src/python/python_targets/BUILD.new'
with create_isolated_git_repo() as worktree:
touch(os.path.join(worktree, new_build_file))
pants_run = self.run_pants(['changed'])
self.assert_success(pants_run)
self.assertEqual(pants_run.stdout_data.strip(), '')
def test_register_copy_file_exists(self):
with distribution() as sdk:
with temporary_dir() as workdir:
android_sdk = AndroidDistribution.cached(sdk)
android_jar = os.path.join("platforms", "android-19", "android.jar")
existing_file = os.path.join(workdir, android_jar)
touch(existing_file)
android_sdk.register_android_tool(android_jar, workdir=workdir)
self.assertIn(android_jar, android_sdk._validated_tools)
def __init__(self, settings): super(Cobertura, self).__init__(settings) options = settings.options self._coverage_datafile = os.path.join(self._settings.coverage_dir, 'cobertura.ser') touch(self._coverage_datafile) self._rootdirs = defaultdict(OrderedSet) self._include_classes = options.coverage_cobertura_include_classes self._exclude_classes = options.coverage_cobertura_exclude_classes self._nothing_to_instrument = True
def __init__(self, task_exports, context): super(Cobertura, self).__init__(task_exports, context) options = task_exports.task_options self._coverage_datafile = os.path.join(self._coverage_dir, 'cobertura.ser') touch(self._coverage_datafile) self._rootdirs = defaultdict(OrderedSet) self._include_classes = options.coverage_cobertura_include_classes self._exclude_classes = options.coverage_cobertura_exclude_classes self._nothing_to_instrument = True
def _create_tiny_git_repo(self):
with temporary_dir() as gitdir,\
temporary_dir() as worktree:
# A tiny little fake git repo we will set up. initialize_repo() requires at least one file.
readme_file = os.path.join(worktree, 'README')
touch(readme_file)
# The contextmanager interface is only necessary if an explicit gitdir is not provided.
with initialize_repo(worktree, gitdir=gitdir) as git:
yield git, worktree, gitdir
def distribution(self, files=None, executables=None):
with temporary_dir() as jdk:
for f in maybe_list(files or ()):
touch(os.path.join(jdk, f))
for exe in maybe_list(executables or (), expected_type=self.EXE):
path = os.path.join(jdk, exe.name)
with safe_open(path, 'w') as fp:
fp.write(exe.contents or '')
chmod_plus_x(path)
yield jdk
def distribution(files=None, executables=None, java_home=None):
with temporary_dir() as dist_root:
for f in maybe_list(files or ()):
touch(os.path.join(dist_root, f))
for executable in maybe_list(executables or (), expected_type=EXE):
path = os.path.join(dist_root, executable.relpath)
with safe_open(path, 'w') as fp:
java_home = os.path.join(dist_root, java_home) if java_home else dist_root
fp.write(executable.contents(java_home))
chmod_plus_x(path)
yield dist_root
def setUp(self):
self.origin = safe_mkdtemp()
with pushd(self.origin):
subprocess.check_call(['git', 'init', '--bare'])
self.gitdir = safe_mkdtemp()
self.worktree = safe_mkdtemp()
self.readme_file = os.path.join(self.worktree, 'README')
with environment_as(GIT_DIR=self.gitdir, GIT_WORK_TREE=self.worktree):
self.init_repo('depot', self.origin)
touch(self.readme_file)
subprocess.check_call(['git', 'add', 'README'])
safe_mkdir(os.path.join(self.worktree, 'dir'))
with open(os.path.join(self.worktree, 'dir', 'f'), 'w') as f:
f.write("file in subdir")
# Make some symlinks
os.symlink('f', os.path.join(self.worktree, 'dir', 'relative-symlink'))
os.symlink('no-such-file', os.path.join(self.worktree, 'dir', 'relative-nonexistent'))
os.symlink('dir/f', os.path.join(self.worktree, 'dir', 'not-absolute\u2764'))
os.symlink('../README', os.path.join(self.worktree, 'dir', 'relative-dotdot'))
os.symlink('dir', os.path.join(self.worktree, 'link-to-dir'))
os.symlink('README/f', os.path.join(self.worktree, 'not-a-dir'))
os.symlink('loop1', os.path.join(self.worktree, 'loop2'))
os.symlink('loop2', os.path.join(self.worktree, 'loop1'))
subprocess.check_call(['git', 'add', 'README', 'dir', 'loop1', 'loop2',
'link-to-dir', 'not-a-dir'])
subprocess.check_call(['git', 'commit', '-am', 'initial commit with decode -> \x81b'])
self.initial_rev = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
subprocess.check_call(['git', 'tag', 'first'])
subprocess.check_call(['git', 'push', '--tags', 'depot', 'master'])
subprocess.check_call(['git', 'branch', '--set-upstream-to', 'depot/master'])
with safe_open(self.readme_file, 'w') as readme:
readme.write('Hello World.\u2764'.encode('utf-8'))
subprocess.check_call(['git', 'commit', '-am', 'Update README.'])
self.current_rev = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
self.clone2 = safe_mkdtemp()
with pushd(self.clone2):
self.init_repo('origin', self.origin)
subprocess.check_call(['git', 'pull', '--tags', 'origin', 'master:master'])
with safe_open(os.path.realpath('README'), 'a') as readme:
readme.write('--')
subprocess.check_call(['git', 'commit', '-am', 'Update README 2.'])
subprocess.check_call(['git', 'push', '--tags', 'origin', 'master'])
self.git = Git(gitdir=self.gitdir, worktree=self.worktree)
def test_no_family(self):
with self.assertRaises(ResolveError):
self.address_mapper.family('a/c')
# Errors are not cached.
with self.assertRaises(ResolveError):
self.address_mapper.family('a/c')
build_file = os.path.join(self.build_root, 'a/c/c.BUILD.json')
touch(build_file)
address_family = self.address_mapper.family('a/c')
self.assertEqual({}, address_family.addressables)
def setUp(self):
self.basedir = safe_mkdtemp()
self.file_list = ['a', 'b', 'c']
self.file_tar = os.path.join(self.basedir, 'test.tar')
with TarFile.open(self.file_tar, mode='w') as tar:
for f in self.file_list:
full_path = os.path.join(self.basedir, f)
touch(full_path)
tar.add(full_path, f)
safe_delete(full_path)
def test_download_zip(self):
go_fetch = self.create_task(self.context())
with temporary_dir() as dest:
with temporary_dir() as src:
touch(os.path.join(src, 'mydir', 'myfile.go'))
zfile = shutil.make_archive(os.path.join(src, 'mydir'), 'zip',
root_dir=src, base_dir='mydir')
go_fetch._download_zip('file://' + zfile, dest)
self.assertTrue(os.path.isfile(os.path.join(dest, 'myfile.go')))
with self.assertRaises(TaskError):
go_fetch._download_zip('file://' + zfile + 'notreal', dest)
def test_via_pants_runner(self):
with temporary_dir() as root:
root = os.path.realpath(root)
touch(os.path.join(root, 'pants'))
with pushd(root):
self.assertEqual(root, BuildRoot().path)
BuildRoot().reset()
child = os.path.join(root, 'one', 'two')
safe_mkdir(child)
with pushd(child):
self.assertEqual(root, BuildRoot().path)
def distribution(files=None, executables=None, java_home=None):
with subsystem_instance(DistributionLocator):
with temporary_dir() as dist_root:
with environment_as(DIST_ROOT=os.path.join(dist_root, java_home) if java_home else dist_root):
for f in maybe_list(files or ()):
touch(os.path.join(dist_root, f))
for executable in maybe_list(executables or (), expected_type=EXE):
path = os.path.join(dist_root, executable.relpath)
with safe_open(path, 'w') as fp:
fp.write(executable.contents or '')
chmod_plus_x(path)
yield dist_root
def test_safe_rm_oldest_items_in_dir_with_excludes(self):
with temporary_dir() as td:
touch(os.path.join(td, 'file1'))
touch(os.path.join(td, 'file2'))
touch(os.path.join(td, 'file3'))
# Time modified is only accurate to second.
time.sleep(1.1)
touch(os.path.join(td, 'file4'))
excludes = [os.path.join(td, 'file1'), os.path.join(td, 'file2')]
safe_rm_oldest_items_in_dir(td, 1, excludes)
self.assertTrue(os.path.exists(os.path.join(td, 'file1')))
self.assertTrue(os.path.exists(os.path.join(td, 'file2')))
self.assertTrue(os.path.exists(os.path.join(td, 'file4')))
self.assertFalse(os.path.exists(os.path.join(td, 'file3')))
def test_sync_binary_dep_links_refreshes_links(self):
c = self.make_target(spec="libC", target_type=GoLibrary)
b = self.make_target(spec="libB", target_type=GoLibrary)
a = self.make_target(spec="libA",
target_type=GoLibrary,
dependencies=[b, c])
lib_binary_map = self._create_lib_binary_map(a, b, c)
a_gopath = self.go_compile.get_gopath(a)
self.go_compile._sync_binary_dep_links(a, a_gopath, lib_binary_map)
# Ensure future links are older than original links by at least 1.5 seconds.
time.sleep(1.5)
# "Modify" b's binary.
touch(lib_binary_map[b])
self.go_compile._sync_binary_dep_links(a, a_gopath, lib_binary_map)
mtime = lambda t: os.lstat(
os.path.join(os.path.join(a_gopath, "pkg", t.address.spec))
).st_mtime
# Make sure c's link was untouched, while b's link was refreshed.
self.assertLessEqual(mtime(c), mtime(b) - 1)
def test_mergetree_existing(self):
with self.tree() as (src, dst):
# Existing empty files
touch(os.path.join(dst, 'c', '1'))
touch(os.path.join(dst, 'a', 'b', '1'))
mergetree(src, dst)
self.assert_tree(
dst,
self.Dir('a'),
self.File.empty('a/2'),
self.Dir('a/b'),
# Existing overlapping file should be overlayed.
self.File('a/b/1', contents='1'),
self.File.empty('a/b/2'),
self.Dir('b'),
self.File('b/1', contents='1'),
self.File.empty('b/2'),
self.Dir('c'),
# Existing non-overlapping file should be preserved.
self.File.empty('c/1'))
def test_symlink_is_not_correct(self) -> None:
with temporary_dir() as tmpdir_1:
source = os.path.join(tmpdir_1, "source")
source2 = os.path.join(tmpdir_1, "source2")
link_path = os.path.join(tmpdir_1, "link")
touch(source)
# link doesnt exist.
self.assertFalse(symlink_is_correct(source, link_path))
touch(link_path)
# link_path exists but isn't a symlink.
self.assertFalse(symlink_is_correct(source, link_path))
os.remove(link_path)
touch(source2)
os.symlink(source2, link_path)
# link_path and source exist, link is not broken, but link points to the wrong place.
self.assertFalse(symlink_is_correct(source, link_path))
def test_round_trip(prefix=None):
with temporary_dir() as fromdir:
safe_mkdir(os.path.join(fromdir, 'a/b/c'))
touch(os.path.join(fromdir, 'a/b/d/e.txt'))
touch(os.path.join(fromdir, 'a/b/d/文件.java'))
touch(os.path.join(fromdir, 'a/b/文件/f.java'))
with temporary_dir() as archivedir:
archive = archiver.create(fromdir, archivedir, 'archive', prefix=prefix)
with temporary_dir() as todir:
archiver.extract(archive, todir)
fromlisting = self._listtree(fromdir, empty_dirs)
if prefix:
fromlisting = set(os.path.join(prefix, x) for x in fromlisting)
if empty_dirs:
fromlisting.add(prefix)
self.assertEqual(fromlisting, self._listtree(todir, empty_dirs))
def test_safe_rm_oldest_items_in_dir(self) -> None:
with temporary_dir() as td:
touch(os.path.join(td, 'file1'))
safe_mkdir(os.path.join(td, 'file2'))
# Time modified is only accurate to second.
time.sleep(1.1)
touch(os.path.join(td, 'file3'))
touch(os.path.join(td, 'file4'))
safe_mkdir(os.path.join(td, 'file5'))
safe_rm_oldest_items_in_dir(td, 3)
self.assertFalse(os.path.exists(os.path.join(td, 'file1')))
self.assertFalse(os.path.exists(os.path.join(td, 'file2')))
self.assertTrue(os.path.exists(os.path.join(td, 'file3')))
self.assertTrue(os.path.exists(os.path.join(td, 'file4')))
self.assertTrue(os.path.exists(os.path.join(td, 'file5')))
def test_round_trip(prefix=None, concurrency_safe=False):
with temporary_dir() as fromdir:
safe_mkdir(os.path.join(fromdir, 'a/b/c'))
touch(os.path.join(fromdir, 'a/b/d/e.txt'))
touch(os.path.join(fromdir, 'a/b/d/文件.java'))
touch(os.path.join(fromdir, 'a/b/文件/f.java'))
with temporary_dir() as archivedir:
archive = archiver.create(fromdir, archivedir, 'archive', prefix=prefix)
# can't use os.path.splitext because 'abc.tar.gz' would return '.gz'.
self.assertTrue(archive.endswith(expected_ext),
'archive filename {0} does not end with expected extension {1}'.format(
archive, expected_ext))
with temporary_dir() as todir:
archiver.extract(archive, todir, concurrency_safe=concurrency_safe)
fromlisting = self._listtree(fromdir, empty_dirs)
if prefix:
fromlisting = set(os.path.join(prefix, x) for x in fromlisting)
if empty_dirs:
fromlisting.add(prefix)
self.assertEqual(fromlisting, self._listtree(todir, empty_dirs))
def test_incremental_caching(self):
with temporary_dir(root_dir=self.pants_workdir) as jar_dir, \
temporary_dir(root_dir=self.pants_workdir) as dist_dir:
self.set_options(pants_distdir=dist_dir)
target = self.make_target(
'java/classpath:java_lib',
target_type=JavaLibrary,
sources=['com/foo/Bar.java'],
)
context = self.context(target_roots=[target])
runtime_classpath = context.products.get_data(
'runtime_classpath',
init_func=ClasspathProducts.init_func(self.pants_workdir))
task = self.create_task(context)
target_classpath_output = os.path.join(dist_dir,
self.options_scope, 'java',
'classpath', 'java_lib')
# Create a classpath entry.
touch(os.path.join(jar_dir, 'z1.jar'))
runtime_classpath.add_for_target(
target, [(None, os.path.join(jar_dir, 'z1.jar'))])
task.execute()
# Check only one symlink and classpath.txt were created.
self.assertEqual(len(os.listdir(target_classpath_output)), 2)
self.assertEqual(
os.path.realpath(
os.path.join(
target_classpath_output,
sorted(os.listdir(target_classpath_output))[0])),
os.path.join(jar_dir, 'z1.jar'))
# Remove the classpath entry.
runtime_classpath.remove_for_target(
target, [(None, os.path.join(jar_dir, 'z1.jar'))])
# Add a different classpath entry
touch(os.path.join(jar_dir, 'z2.jar'))
runtime_classpath.add_for_target(
target, [(None, os.path.join(jar_dir, 'z2.jar'))])
task.execute()
# Check the symlink was updated.
self.assertEqual(len(os.listdir(target_classpath_output)), 2)
self.assertEqual(
os.path.realpath(
os.path.join(
target_classpath_output,
sorted(os.listdir(target_classpath_output))[0])),
os.path.join(jar_dir, 'z2.jar'))
# Add a different classpath entry
touch(os.path.join(jar_dir, 'z3.jar'))
runtime_classpath.add_for_target(
target, [(None, os.path.join(jar_dir, 'z3.jar'))])
task.execute()
self.assertEqual(len(os.listdir(target_classpath_output)), 3)
classpath = sorted(os.listdir(target_classpath_output))[2]
with safe_open(os.path.join(target_classpath_output,
classpath)) as classpath_file:
# Assert there is only one line ending with a newline
self.assertListEqual(classpath_file.readlines(), [
os.pathsep.join([
os.path.join(jar_dir, 'z2.jar'),
os.path.join(jar_dir, 'z3.jar')
]) + '\n'
])
def join():
touch(await_file)
return waiter_handle.join()
def setUp(self):
self.origin = safe_mkdtemp()
with pushd(self.origin):
subprocess.check_call(['git', 'init', '--bare'])
self.gitdir = safe_mkdtemp()
self.worktree = safe_mkdtemp()
self.readme_file = os.path.join(self.worktree, 'README')
with environment_as(GIT_DIR=self.gitdir, GIT_WORK_TREE=self.worktree):
self.init_repo('depot', self.origin)
touch(self.readme_file)
subprocess.check_call(['git', 'add', 'README'])
safe_mkdir(os.path.join(self.worktree, 'dir'))
with open(os.path.join(self.worktree, 'dir', 'f'), 'w') as f:
f.write("file in subdir")
# Make some symlinks
os.symlink('f',
os.path.join(self.worktree, 'dir', 'relative-symlink'))
os.symlink(
'no-such-file',
os.path.join(self.worktree, 'dir', 'relative-nonexistent'))
os.symlink(
'dir/f',
os.path.join(self.worktree, 'dir', 'not-absolute\u2764'))
os.symlink('../README',
os.path.join(self.worktree, 'dir', 'relative-dotdot'))
os.symlink('dir', os.path.join(self.worktree, 'link-to-dir'))
os.symlink('README/f', os.path.join(self.worktree, 'not-a-dir'))
os.symlink('loop1', os.path.join(self.worktree, 'loop2'))
os.symlink('loop2', os.path.join(self.worktree, 'loop1'))
subprocess.check_call([
'git', 'add', 'README', 'dir', 'loop1', 'loop2', 'link-to-dir',
'not-a-dir'
])
subprocess.check_call([
'git', 'commit', '-am', 'initial commit with decode -> \x81b'
])
self.initial_rev = subprocess.check_output(
['git', 'rev-parse', 'HEAD']).strip()
subprocess.check_call(['git', 'tag', 'first'])
subprocess.check_call(['git', 'push', '--tags', 'depot', 'master'])
subprocess.check_call(
['git', 'branch', '--set-upstream-to', 'depot/master'])
with safe_open(self.readme_file, 'wb') as readme:
readme.write('Hello World.\u2764'.encode())
subprocess.check_call(['git', 'commit', '-am', 'Update README.'])
self.current_rev = subprocess.check_output(
['git', 'rev-parse', 'HEAD']).strip()
self.clone2 = safe_mkdtemp()
with pushd(self.clone2):
self.init_repo('origin', self.origin)
subprocess.check_call(
['git', 'pull', '--tags', 'origin', 'master:master'])
with safe_open(os.path.realpath('README'), 'a') as readme:
readme.write('--')
subprocess.check_call(['git', 'commit', '-am', 'Update README 2.'])
subprocess.check_call(
['git', 'push', '--tags', 'origin', 'master'])
self.git = Git(gitdir=self.gitdir, worktree=self.worktree)
def test_overwrite_file(self) -> None:
touch(self.link)
self._create_and_check_link(self.source, self.link)
def test_safe_rm_oldest_items_in_dir_noop(self):
with temporary_dir() as td:
safe_rm_oldest_items_in_dir(td, 1)
touch(os.path.join(td, 'file1'))
self.assertEqual(len(os.listdir(td)), 1)
def touch(cls, path): touch(os.path.join(BuildFileTest.root_dir, path))
def _create_binary(self, target):
p = os.path.join(self.go_compile.get_gopath(target), 'pkg',
target.address.spec)
touch(p)
return p
def generate_class(name):
path = os.path.join(self.classes_dir, name)
touch(path)
return path
def _create_result_file(self, target):
result_file = os.path.join(self._results_dir, target.id)
touch(result_file)
return result_file
def instrument(self, output_dir):
for datafile in self._iter_datafiles(output_dir):
os.unlink(datafile)
self._canonical_datafile = os.path.join(
output_dir, '{}.canonical'.format(self._DATAFILE_NAME))
# It's conceivable we'll be executing a test that has no source file dependencies; ie: we'll
# never generate a canonical coverage datafile below. Create an empty one here to allow the
# test run to proceeed normally.
touch(self._canonical_datafile)
# Setup an instrumentation classpath based on the existing runtime classpath.
runtime_classpath = self._context.products.get_data(
'runtime_classpath')
instrumentation_classpath = self._context.products.safe_create_data(
'instrument_classpath', runtime_classpath.copy)
self.initialize_instrument_classpath(output_dir, self._settings,
self._targets,
instrumentation_classpath)
cobertura_cp = self._settings.tool_classpath('cobertura-instrument')
files_to_instrument = []
for target in self._targets:
if Cobertura.is_coverage_target(target):
paths = instrumentation_classpath.get_for_target(target)
for (name, path) in paths:
files_to_instrument.append(path)
if len(files_to_instrument) > 0:
unique_files = list(set(files_to_instrument))
relativize_paths(unique_files, self._settings.workdir)
args = [
'--basedir',
self._settings.workdir,
'--datafile',
self._canonical_datafile,
]
if self._include_user_classpath:
closure = BuildGraph.closure(
self._targets,
bfs=True,
include_scopes=Scopes.JVM_TEST_SCOPES,
respect_intransitive=True)
aux_classpath = safe_classpath(ClasspathUtil.classpath(
closure, runtime_classpath),
synthetic_jar_dir=None)
args.append('--auxClasspath')
args.extend(aux_classpath)
# apply class incl/excl filters
if len(self._include_classes) > 0:
for pattern in self._include_classes:
args += ["--includeClasses", pattern]
else:
args += ["--includeClasses",
'.*'] # default to instrumenting all classes
for pattern in self._exclude_classes:
args += ["--excludeClasses", pattern]
with temporary_file(binary_mode=False) as tmp_file:
tmp_file.write("\n".join(unique_files))
tmp_file.flush()
args += ["--listOfFilesToInstrument", tmp_file.name]
main = 'net.sourceforge.cobertura.instrument.InstrumentMain'
self._settings.log.debug(
"executing cobertura instrumentation with the following args: {}"
.format(args))
result = self._execute_java(
classpath=cobertura_cp,
main=main,
jvm_options=self._settings.coverage_jvm_options,
args=args,
workunit_factory=self._context.new_workunit,
workunit_name='cobertura-instrument')
if result != 0:
raise TaskError("java {0} ... exited non-zero ({1})"
" 'failed to instrument'".format(
main, result))
def test_changed_with_multiple_build_files(self):
new_build_file = "src/python/python_targets/BUILD.new"
with create_isolated_git_repo() as worktree:
touch(os.path.join(worktree, new_build_file))
stdout_data = self.run_list()
self.assertEqual(stdout_data.strip(), "")
def test_rm_rf_no_such_file_not_an_error(self, file_name='./vanishing_file'):
with temporary_dir() as td, pushd(td), \
mock.patch('pants.util.dirutil.shutil.rmtree') as mock_rmtree:
mock_rmtree.side_effect = OSError(errno.ENOENT, os.strerror(errno.ENOENT))
touch(file_name)
rm_rf(file_name)
def setUp(self):
self.origin = safe_mkdtemp()
with pushd(self.origin):
subprocess.check_call(["git", "init", "--bare"])
self.gitdir = safe_mkdtemp()
self.worktree = safe_mkdtemp()
self.readme_file = os.path.join(self.worktree, "README")
with environment_as(GIT_DIR=self.gitdir, GIT_WORK_TREE=self.worktree):
self.init_repo("depot", self.origin)
touch(self.readme_file)
subprocess.check_call(["git", "add", "README"])
safe_mkdir(os.path.join(self.worktree, "dir"))
with open(os.path.join(self.worktree, "dir", "f"), "w") as f:
f.write("file in subdir")
# Make some symlinks
os.symlink("f",
os.path.join(self.worktree, "dir", "relative-symlink"))
os.symlink(
"no-such-file",
os.path.join(self.worktree, "dir", "relative-nonexistent"))
os.symlink(
"dir/f",
os.path.join(self.worktree, "dir", "not-absolute\u2764"))
os.symlink("../README",
os.path.join(self.worktree, "dir", "relative-dotdot"))
os.symlink("dir", os.path.join(self.worktree, "link-to-dir"))
os.symlink("README/f", os.path.join(self.worktree, "not-a-dir"))
os.symlink("loop1", os.path.join(self.worktree, "loop2"))
os.symlink("loop2", os.path.join(self.worktree, "loop1"))
subprocess.check_call([
"git", "add", "README", "dir", "loop1", "loop2", "link-to-dir",
"not-a-dir"
])
subprocess.check_call([
"git", "commit", "-am", "initial commit with decode -> \x81b"
])
self.initial_rev = subprocess.check_output(
["git", "rev-parse", "HEAD"]).strip()
subprocess.check_call(["git", "tag", "first"])
subprocess.check_call(["git", "push", "--tags", "depot", "master"])
subprocess.check_call(
["git", "branch", "--set-upstream-to", "depot/master"])
with safe_open(self.readme_file, "wb") as readme:
readme.write("Hello World.\u2764".encode())
subprocess.check_call(["git", "commit", "-am", "Update README."])
self.current_rev = subprocess.check_output(
["git", "rev-parse", "HEAD"]).strip()
self.clone2 = safe_mkdtemp()
with pushd(self.clone2):
self.init_repo("origin", self.origin)
subprocess.check_call(
["git", "pull", "--tags", "origin", "master:master"])
with safe_open(os.path.realpath("README"), "a") as readme:
readme.write("--")
subprocess.check_call(["git", "commit", "-am", "Update README 2."])
subprocess.check_call(
["git", "push", "--tags", "origin", "master"])
self.git = Git(gitdir=self.gitdir, worktree=self.worktree)
def fixup_times(path): touch(path, times=(source_mtime, source_mtime))
def create_binary(name):
target = self.make_target(name, GoBinary)
executable = os.path.join(bin_source_dir,
'{}.exe'.format(name))
touch(executable)
return target, executable
def test_rm_rf_file(self, file_name="./foo") -> None:
with temporary_dir() as td, pushd(td):
touch(file_name)
self.assertTrue(os.path.isfile(file_name))
rm_rf(file_name)
self.assertFalse(os.path.exists(file_name))
def touch(self, path):
touch(self.fullpath(path))
def file_toucher():
while not halt.isSet():
touch(f)
time.sleep(1)
def test_mergetree_existing_file_mismatch(self):
with self.tree() as (src, dst):
touch(os.path.join(dst, 'a'))
with self.assertRaises(ExistingFileError):
mergetree(src, dst)
def unexecutable_git(self):
with self.empty_path() as path:
git = os.path.join(path, 'git')
touch(git)
yield git
def create_isolated_git_repo():
# Isolated Git Repo Structure:
# worktree
# |--README
# |--pants.ini
# |--3rdparty
# |--BUILD
# |--src
# |--resources
# |--org/pantsbuild/resourceonly
# |--BUILD
# |--README.md
# |--java
# |--org/pantsbuild/helloworld
# |--BUILD
# |--helloworld.java
# |--python
# |--python_targets
# |--BUILD
# |--test_binary.py
# |--test_library.py
# |--test_unclaimed_src.py
# |--sources
# |--BUILD
# |--sources.py
# |--sources.txt
# |--tests
# |--scala
# |--org/pantsbuild/cp-directories
# |--BUILD
# |--ClasspathDirectories.scala
with temporary_dir(root_dir=get_buildroot()) as worktree:
with safe_open(os.path.join(worktree, 'README'), 'w') as fp:
fp.write('Just a test tree.')
# Create an empty pants config file.
touch(os.path.join(worktree, 'pants.ini'))
# Copy .gitignore to new repo.
shutil.copyfile('.gitignore', os.path.join(worktree, '.gitignore'))
with initialize_repo(worktree=worktree,
gitdir=os.path.join(worktree, '.git')) as git:
# Resource File
resource_file = os.path.join(
worktree,
'src/resources/org/pantsbuild/resourceonly/README.md')
with safe_open(resource_file, 'w') as fp:
fp.write('Just resource.')
resource_build_file = os.path.join(
worktree, 'src/resources/org/pantsbuild/resourceonly/BUILD')
with safe_open(resource_build_file, 'w') as fp:
fp.write(
dedent("""
resources(
name='resource',
sources=['README.md'],
)
"""))
git.add(resource_file, resource_build_file)
git.commit('Check in a resource target.')
# Java Program
src_file = os.path.join(
worktree, 'src/java/org/pantsbuild/helloworld/helloworld.java')
with safe_open(src_file, 'w') as fp:
fp.write(
dedent("""
package org.pantsbuild.helloworld;
class HelloWorld {
public static void main(String[] args) {
System.out.println("Hello, World!\n");
}
}
"""))
src_build_file = os.path.join(
worktree, 'src/java/org/pantsbuild/helloworld/BUILD')
with safe_open(src_build_file, 'w') as fp:
fp.write(
dedent("""
jvm_binary(
dependencies=[
'{}',
],
source='helloworld.java',
main='org.pantsbuild.helloworld.HelloWorld',
)
""".format('src/resources/org/pantsbuild/resourceonly:resource')))
git.add(src_file, src_build_file)
git.commit(
'hello world java program with a dependency on a resource file.'
)
# Scala Program
scala_src_dir = os.path.join(
worktree, 'tests/scala/org/pantsbuild/cp-directories')
safe_mkdir(os.path.dirname(scala_src_dir))
shutil.copytree(
'testprojects/tests/scala/org/pantsbuild/testproject/cp-directories',
scala_src_dir)
git.add(scala_src_dir)
git.commit('Check in a scala test target.')
# Python library and binary
python_src_dir = os.path.join(worktree,
'src/python/python_targets')
safe_mkdir(os.path.dirname(python_src_dir))
shutil.copytree('testprojects/src/python/python_targets',
python_src_dir)
git.add(python_src_dir)
git.commit('Check in python targets.')
# A `python_library` with `resources=['file.name']`.
python_src_dir = os.path.join(worktree, 'src/python/sources')
safe_mkdir(os.path.dirname(python_src_dir))
shutil.copytree('testprojects/src/python/sources', python_src_dir)
git.add(python_src_dir)
git.commit('Check in a python library with resource dependency.')
# Copy 3rdparty/BUILD.
_3rdparty_build = os.path.join(worktree, '3rdparty/BUILD')
safe_mkdir(os.path.dirname(_3rdparty_build))
shutil.copyfile('3rdparty/BUILD', _3rdparty_build)
git.add(_3rdparty_build)
git.commit('Check in 3rdparty/BUILD.')
with environment_as(PANTS_BUILDROOT_OVERRIDE=worktree):
yield worktree
def prepare_resources(self, target, chroot):
for i in range(self.get_target_ordinal(target)):
touch(os.path.join(chroot, target.id, str(i)))
def _create_mock_build_file(self, dirname):
with temporary_dir() as root:
os.mkdir(os.path.join(root, dirname))
touch(os.path.join(root, dirname, 'BUILD'))
yield BuildFile(FileSystemProjectTree(root),
os.path.join(dirname, 'BUILD'))
