def test_analysis_portability(self):
# Tests that analysis can be relocated between workdirs and still result in incremental
# compile.
with temporary_dir() as cache_dir, temporary_dir(root_dir=get_buildroot()) as src_dir, \
temporary_dir(root_dir=get_buildroot(), suffix='.pants.d') as workdir:
config = {
'cache.compile.zinc': {'write_to': [cache_dir], 'read_from': [cache_dir]},
}
dep_src_file = os.path.join(src_dir, 'org', 'pantsbuild', 'dep', 'A.scala')
dep_build_file = os.path.join(src_dir, 'org', 'pantsbuild', 'dep', 'BUILD')
con_src_file = os.path.join(src_dir, 'org', 'pantsbuild', 'consumer', 'B.scala')
con_build_file = os.path.join(src_dir, 'org', 'pantsbuild', 'consumer', 'BUILD')
dep_spec = os.path.join(os.path.basename(src_dir), 'org', 'pantsbuild', 'dep')
con_spec = os.path.join(os.path.basename(src_dir), 'org', 'pantsbuild', 'consumer')
dep_src = "package org.pantsbuild.dep; class A {}"
self.create_file(dep_src_file, dep_src)
self.create_file(dep_build_file, "scala_library()")
self.create_file(con_src_file, dedent(
"""package org.pantsbuild.consumer
import org.pantsbuild.dep.A
class B { def mkA: A = new A() }"""))
self.create_file(con_build_file, "scala_library(dependencies=['{}'])".format(dep_spec))
rel_workdir = fast_relpath(workdir, get_buildroot())
rel_src_dir = fast_relpath(src_dir, get_buildroot())
with self.mock_buildroot(dirs_to_copy=[rel_src_dir, rel_workdir]) as buildroot, \
buildroot.pushd():
# 1) Compile in one buildroot.
self.run_compile(con_spec, config, os.path.join(buildroot.new_buildroot, rel_workdir))
with self.mock_buildroot(dirs_to_copy=[rel_src_dir, rel_workdir]) as buildroot, \
buildroot.pushd():
# 2) Compile in another buildroot, and check that we hit the cache.
new_workdir = os.path.join(buildroot.new_buildroot, rel_workdir)
run_two = self.run_compile(con_spec, config, new_workdir)
self.assertTrue(
re.search(
"\[zinc\][^[]*\[cache\][^[]*Using cached artifacts for 2 targets.",
run_two.stdout_data),
run_two.stdout_data)
# 3) Edit the dependency in a way that should trigger an incremental
# compile of the consumer.
mocked_dep_src_file = os.path.join(
buildroot.new_buildroot,
fast_relpath(dep_src_file, get_buildroot()))
self.create_file(mocked_dep_src_file, dep_src + "; /* this is a comment */")
# 4) Compile and confirm that the analysis fetched from the cache in
# step 2 causes incrementalism: ie, zinc does not report compiling any files.
run_three = self.run_compile(con_spec, config, new_workdir)
self.assertTrue(
re.search(
r"/org/pantsbuild/consumer:consumer\)[^[]*\[compile\][^[]*\[zinc\]\W*\[info\] Compile success",
run_three.stdout_data),
run_three.stdout_data)
def __init__(self, *args, **kwargs):
super(BaseZincCompile, self).__init__(*args, **kwargs)
# A directory to contain per-target subdirectories with apt processor info files.
self._processor_info_dir = os.path.join(self.workdir, 'apt-processor-info')
# Validate zinc options.
ZincCompile.validate_arguments(self.context.log, self.get_options().whitelisted_args,
self._args)
if self.execution_strategy == self.HERMETIC:
# TODO: Make incremental compiles work. See:
# hermetically https://github.com/pantsbuild/pants/issues/6517
if self.get_options().incremental:
raise TaskError("Hermetic zinc execution does not currently support incremental compiles. "
"Please use --no-compile-zinc-incremental.")
try:
fast_relpath(self.get_options().pants_workdir, get_buildroot())
except ValueError:
raise TaskError(
"Hermetic zinc execution currently requires the workdir to be a child of the buildroot "
"but workdir was {} and buildroot was {}".format(
self.get_options().pants_workdir,
get_buildroot(),
)
)
if self.get_options().use_classpath_jars:
# TODO: Make this work by capturing the correct Digest and passing them around the
# right places.
# See https://github.com/pantsbuild/pants/issues/6432
raise TaskError("Hermetic zinc execution currently doesn't work with classpath jars")
def __init__(self, project_tree, relpath, must_exist=True):
"""Creates a BuildFile object representing the BUILD file family at the specified path.
:param project_tree: Project tree the BUILD file exist in.
:type project_tree: :class:`pants.base.project_tree.ProjectTree`
:param string relpath: The path relative to root_dir where the BUILD file is found - this can
either point directly at the BUILD file or else to a directory which contains BUILD files.
:param bool must_exist: If True, at least one BUILD file must exist at the given location or
else an` `MissingBuildFileError` is thrown
:raises IOError: if the root_dir path is not absolute.
:raises MissingBuildFileError: if the path does not house a BUILD file and must_exist is `True`.
"""
self.project_tree = project_tree
self.root_dir = project_tree.build_root
path = os.path.join(self.root_dir, relpath) if relpath else self.root_dir
self._build_basename = self._BUILD_FILE_PREFIX
if project_tree.isdir(fast_relpath(path, self.root_dir)):
buildfile = os.path.join(path, self._build_basename)
else:
buildfile = path
# There is no BUILD file without a prefix so select any viable sibling
buildfile_relpath = fast_relpath(buildfile, self.root_dir)
if not project_tree.exists(buildfile_relpath) or project_tree.isdir(buildfile_relpath):
for build in self._get_all_build_files(os.path.dirname(buildfile)):
self._build_basename = build
buildfile = os.path.join(path, self._build_basename)
buildfile_relpath = fast_relpath(buildfile, self.root_dir)
break
if must_exist:
if not project_tree.exists(buildfile_relpath):
raise self.MissingBuildFileError('BUILD file does not exist at: {path}'
.format(path=buildfile))
# If a build file must exist then we want to make sure it's not a dir.
# In other cases we are ok with it being a dir, for example someone might have
# repo/scripts/build/doit.sh.
if project_tree.isdir(buildfile_relpath):
raise self.MissingBuildFileError('Path to buildfile ({buildfile}) is a directory, '
'but it must be a file.'.format(buildfile=buildfile))
if not self._is_buildfile_name(os.path.basename(buildfile)):
raise self.MissingBuildFileError('{path} is not a BUILD file'
.format(path=buildfile))
self.full_path = os.path.realpath(buildfile)
self.name = os.path.basename(self.full_path)
self.parent_path = os.path.dirname(self.full_path)
self.relpath = fast_relpath(self.full_path, self.root_dir)
self.spec_path = os.path.dirname(self.relpath)
def walk(self, relpath, topdown=True):
"""
Walk the file tree rooted at `path`. Works like os.walk but returned root value is relative path.
Ignored paths will not be returned.
"""
for root, dirs, files in self._walk_raw(relpath, topdown):
matched_dirs = self.ignore.match_files([os.path.join(root, "{}/".format(d)) for d in dirs])
matched_files = self.ignore.match_files([os.path.join(root, f) for f in files])
for matched_dir in matched_dirs:
dirs.remove(fast_relpath(matched_dir, root).rstrip('/'))
for matched_file in matched_files:
files.remove(fast_relpath(matched_file, root))
yield root, dirs, files
def scan_addresses(self, root=None):
"""Recursively gathers all addresses visible under `root` of the virtual address space.
:param string root: The absolute path of the root to scan; defaults to the root directory of the
pants project.
:rtype: set of :class:`pants.build_graph.address.Address`
:raises AddressLookupError: if there is a problem parsing a BUILD file
"""
root_dir = get_buildroot()
base_path = None
if root:
try:
base_path = fast_relpath(root, root_dir)
except ValueError as e:
raise self.InvalidRootError(e)
addresses = set()
try:
for build_file in BuildFile.scan_build_files(self._project_tree,
base_relpath=base_path,
build_ignore_patterns=self._build_ignore_patterns):
for address in self.addresses_in_spec_path(build_file.spec_path):
addresses.add(address)
except BuildFile.BuildFileError as e:
# Handle exception from BuildFile out of paranoia. Currently, there is no way to trigger it.
raise self.BuildFileScanError("{message}\n while scanning BUILD files in '{root}'."
.format(message=e, root=root))
return addresses
def create_fileset_with_spec(cls, rel_path, *patterns, **kwargs):
"""
:param rel_path: The relative path to create a FilesetWithSpec for.
:param patterns: glob patterns to apply.
:param exclude: A list of {,r,z}globs objects, strings, or lists of strings to exclude.
"""
for pattern in patterns:
if not isinstance(pattern, string_types):
raise ValueError("Expected string patterns for {}: got {}".format(cls.__name__, patterns))
raw_exclude = kwargs.pop('exclude', [])
buildroot = get_buildroot()
root = os.path.normpath(os.path.join(buildroot, rel_path))
# making sure there are no unknown arguments.
unknown_args = set(kwargs.keys()) - cls.KNOWN_PARAMETERS
if unknown_args:
raise ValueError('Unexpected arguments while parsing globs: {}'.format(
', '.join(unknown_args)))
for glob in patterns:
if cls._is_glob_dir_outside_root(glob, root):
raise ValueError('Invalid glob {}, points outside BUILD file root {}'.format(glob, root))
exclude = cls.process_raw_exclude(raw_exclude)
files_calculator = cls._file_calculator(root, patterns, kwargs, exclude)
rel_root = fast_relpath(root, buildroot)
if rel_root == '.':
rel_root = ''
filespec = cls.to_filespec(patterns, root=rel_root, exclude=exclude)
return LazyFilesetWithSpec(rel_root, filespec, files_calculator)
def walk(self, relpath, topdown=True):
def onerror(error):
raise OSError(getattr(error, 'errno', None), 'Failed to walk below {}'.format(relpath), error)
for root, dirs, files in safe_walk(os.path.join(self.build_root, relpath),
topdown=topdown,
onerror=onerror):
yield fast_relpath(root, self.build_root), dirs, files
def scan_build_files(project_tree, base_relpath, build_ignore_patterns=None):
"""Looks for all BUILD files
:param project_tree: Project tree to scan in.
:type project_tree: :class:`pants.base.project_tree.ProjectTree`
:param base_relpath: Directory under root_dir to scan.
:param build_ignore_patterns: .gitignore like patterns to exclude from BUILD files scan.
:type build_ignore_patterns: pathspec.pathspec.PathSpec
"""
if base_relpath and os.path.isabs(base_relpath):
raise BuildFile.BadPathError('base_relpath parameter ({}) should be a relative path.'
.format(base_relpath))
if base_relpath and not project_tree.isdir(base_relpath):
raise BuildFile.BadPathError('Can only scan directories and {0} is not a valid dir.'
.format(base_relpath))
if build_ignore_patterns and not isinstance(build_ignore_patterns, PathSpec):
raise TypeError("build_ignore_patterns should be pathspec.pathspec.PathSpec instance, "
"instead {} was given.".format(type(build_ignore_patterns)))
build_files = set()
for root, dirs, files in project_tree.walk(base_relpath or '', topdown=True):
excluded_dirs = list(build_ignore_patterns.match_files('{}/'.format(os.path.join(root, dirname))
for dirname in dirs))
for subdir in excluded_dirs:
# Remove trailing '/' from paths which were added to indicate that paths are paths to directories.
dirs.remove(fast_relpath(subdir, root)[:-1])
for filename in files:
if BuildFile._is_buildfile_name(filename):
build_files.add(os.path.join(root, filename))
return BuildFile._build_files_from_paths(project_tree, build_files, build_ignore_patterns)
def _record_compile_classpath(self, classpath, target, outdir):
relative_classpaths = [fast_relpath(path, self.get_options().pants_workdir) for path in classpath]
text = '\n'.join(relative_classpaths)
path = os.path.join(outdir, 'compile_classpath', '{}.txt'.format(target.id))
safe_mkdir(os.path.dirname(path), clean=False)
with open(path, 'w') as f:
f.write(text)
def _get_all_build_files(self, path):
"""Returns all the BUILD files on a path"""
results = []
relpath = fast_relpath(path, self.root_dir)
for build in self.project_tree.glob1(relpath, '{prefix}*'.format(prefix=self._BUILD_FILE_PREFIX)):
if self._is_buildfile_name(build) and self.project_tree.isfile(os.path.join(relpath, build)):
results.append(build)
return sorted(results)
def relativize(paths, project_tree):
for path in paths:
if os.path.isabs(path):
realpath = os.path.realpath(path)
if realpath.startswith(project_tree.build_root):
yield fast_relpath(realpath, project_tree.build_root)
else:
yield path
def siblings(self):
"""Returns an iterator over all the BUILD files co-located with this BUILD file not including
this BUILD file itself"""
for build in BuildFile.get_build_files_family(self.project_tree,
fast_relpath(self.parent_path, self.root_dir)):
if self != build:
yield build
def _tool_classpath(self, tool, products, scheduler):
"""Return the proper classpath based on products and scala version."""
classpath = self.tool_classpath_from_products(products,
self.versioned_tool_name(tool, self.version),
scope=self.options_scope)
classpath = tuple(fast_relpath(c, get_buildroot()) for c in classpath)
return self._memoized_scalac_classpath(classpath, scheduler)
def _compile_hermetic(self, jvm_options, ctx, classes_dir, zinc_args,
compiler_bridge_classpath_entry, dependency_classpath,
scalac_classpath_entries):
zinc_relpath = fast_relpath(self._zinc.zinc, get_buildroot())
snapshots = [
self._zinc.snapshot(self.context._scheduler),
ctx.target.sources_snapshot(self.context._scheduler),
]
relevant_classpath_entries = dependency_classpath + [compiler_bridge_classpath_entry]
directory_digests = tuple(
entry.directory_digest for entry in relevant_classpath_entries if entry.directory_digest
)
if len(directory_digests) != len(relevant_classpath_entries):
for dep in relevant_classpath_entries:
if dep.directory_digest is None:
logger.warning(
"ClasspathEntry {} didn't have a Digest, so won't be present for hermetic "
"execution".format(dep)
)
snapshots.extend(
classpath_entry.directory_digest for classpath_entry in scalac_classpath_entries
)
# TODO: Extract something common from Executor._create_command to make the command line
# TODO: Lean on distribution for the bin/java appending here
merged_input_digest = self.context._scheduler.merge_directories(
tuple(s.directory_digest for s in snapshots) + directory_digests
)
argv = ['.jdk/bin/java'] + jvm_options + [
'-cp', zinc_relpath,
Zinc.ZINC_COMPILE_MAIN
] + zinc_args
# TODO(#6071): Our ExecuteProcessRequest expects a specific string type for arguments,
# which py2 doesn't default to. This can be removed when we drop python 2.
argv = [text_type(arg) for arg in argv]
req = ExecuteProcessRequest(
argv=tuple(argv),
input_files=merged_input_digest,
output_directories=(classes_dir,),
description="zinc compile for {}".format(ctx.target.address.spec),
# TODO: These should always be unicodes
# Since this is always hermetic, we need to use `underlying_dist`
jdk_home=text_type(self._zinc.underlying_dist.home),
)
res = self.context.execute_process_synchronously_or_raise(
req, self.name(), [WorkUnitLabel.COMPILER])
# TODO: Materialize as a batch in do_compile or somewhere
self.context._scheduler.materialize_directories((
DirectoryToMaterialize(get_buildroot(), res.output_directory_digest),
))
# TODO: This should probably return a ClasspathEntry rather than a Digest
return res.output_directory_digest
def _spec_excludes_to_gitignore_syntax(build_root, spec_excludes=None):
if spec_excludes:
for path in spec_excludes:
if os.path.isabs(path):
realpath = os.path.realpath(path)
if realpath.startswith(build_root):
yield '/{}'.format(fast_relpath(realpath, build_root))
else:
yield '/{}'.format(path)
def ancestors(self):
"""Returns all BUILD files in ancestor directories of this BUILD file's parent directory."""
parent_buildfiles = OrderedSet()
parentdir = fast_relpath(self.parent_path, self.root_dir)
while parentdir != '':
parentdir = os.path.dirname(parentdir)
parent_buildfiles.update(BuildFile.get_build_files_family(self.project_tree, parentdir))
return parent_buildfiles
def descendants(self, spec_excludes=None):
"""Returns all BUILD files in descendant directories of this BUILD file's parent directory."""
descendants = self.scan_project_tree_build_files(self.project_tree,
fast_relpath(self.parent_path, self.root_dir),
spec_excludes=spec_excludes)
for sibling in self.family():
descendants.discard(sibling)
return descendants
def scan_addresses(self, root=None):
if root:
try:
base_path = fast_relpath(root, self._build_root)
except ValueError as e:
raise self.InvalidRootError(e)
else:
base_path = ''
return self._internal_scan_specs([DescendantAddresses(base_path)], missing_is_fatal=False)
def hydrate_sources(sources_field, source_files_content, excluded_source_files):
"""Given a SourcesField and FilesContent for its path_globs, create an EagerFilesetWithSpec."""
excluded = {f.path for f in excluded_source_files.dependencies}
spec_path = sources_field.address.spec_path
filespecs = sources_field.filespecs
print('>>> filespecs are {}; included/excluded are: {} / {}'.format(filespecs, [fc.path for fc in source_files_content.dependencies], excluded))
file_hashes = {fast_relpath(fc.path, spec_path): sha1(fc.content).digest()
for fc in source_files_content.dependencies
if fc.path not in excluded}
return HydratedField('sources', EagerFilesetWithSpec(spec_path, filespecs, file_hashes))
def scan_addresses(self, root=None):
if root:
try:
base_path = fast_relpath(root, self._build_root)
except ValueError as e:
raise self.InvalidRootError(e)
else:
base_path = ""
return self.scan_specs([DescendantAddresses(base_path)])
def render_logs(workdir):
"""Renders all potentially relevant logs from the given workdir to stdout."""
filenames = list(glob.glob(os.path.join(
workdir, 'logs/exceptions*log'))) + list(
glob.glob(os.path.join(workdir, 'pantsd/pantsd.log')))
for filename in filenames:
rel_filename = fast_relpath(filename, workdir)
print('{} +++ '.format(rel_filename))
for line in _read_log(filename):
print('{} >>> {}'.format(rel_filename, line))
print('{} --- '.format(rel_filename))
def _snapshotted_classpath(self, results_dir):
relpath = fast_relpath(results_dir, get_buildroot())
(classes_dir_snapshot, ) = self.context._scheduler.capture_snapshots([
PathGlobsAndRoot(
PathGlobs([relpath]),
get_buildroot(),
Digest.load(relpath),
),
])
return ClasspathEntry(results_dir,
classes_dir_snapshot.directory_digest)
def _record_compile_classpath(self, classpath, target, outdir):
relative_classpaths = [
fast_relpath(path,
self.get_options().pants_workdir)
for path in classpath
]
text = '\n'.join(relative_classpaths)
path = os.path.join(outdir, 'compile_classpath', f'{target.id}.txt')
safe_mkdir(os.path.dirname(path), clean=False)
with open(path, 'w') as f:
f.write(text)
def readlink(self, relpath):
link_path = self.relative_readlink(relpath)
if os.path.isabs(link_path):
raise IOError('Absolute symlinks not supported in {}: {} -> {}'.format(
self, relpath, link_path))
# In order to enforce that this link does not escape the build_root, we join and
# then remove it.
abs_normpath = os.path.normpath(os.path.join(self.build_root,
os.path.dirname(relpath),
link_path))
return fast_relpath(abs_normpath, self.build_root)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# A directory to contain per-target subdirectories with apt processor info files.
self._processor_info_dir = os.path.join(self.workdir,
'apt-processor-info')
# Validate zinc options.
ZincCompile.validate_arguments(self.context.log,
self.get_options().whitelisted_args,
self._args)
if self.execution_strategy == self.ExecutionStrategy.hermetic:
try:
fast_relpath(self.get_options().pants_workdir, get_buildroot())
except ValueError:
raise TaskError(
"Hermetic zinc execution currently requires the workdir to be a child of the buildroot "
"but workdir was {} and buildroot was {}".format(
self.get_options().pants_workdir,
get_buildroot(),
))
def scan_addresses(self, root=None):
if root:
try:
base_path = fast_relpath(root, self._build_root)
except ValueError as e:
raise self.InvalidRootError(e)
else:
base_path = ''
return self._internal_scan_specs([DescendantAddresses(base_path)],
missing_is_fatal=False)
def _eager_fileset_with_spec(spec_path, filespecs, source_files_digest, excluded_source_files):
excluded = {f.path for f in excluded_source_files.dependencies}
file_tuples = [(fast_relpath(fd.path, spec_path), fd.digest)
for fd in source_files_digest.dependencies
if fd.path not in excluded]
# NB: In order to preserve declared ordering, we record a list of matched files
# independent of the file hash dict.
return EagerFilesetWithSpec(spec_path,
filespecs,
files=tuple(f for f, _ in file_tuples),
file_hashes=dict(file_tuples))
def descendants(self, spec_excludes=None):
"""Returns all BUILD files in descendant directories of this BUILD file's parent directory."""
descendants = BuildFile.scan_build_files(self.project_tree,
fast_relpath(
self.parent_path,
self.root_dir),
spec_excludes=spec_excludes)
for sibling in self.family():
descendants.discard(sibling)
return descendants
def filter_ignored(self, path_list, prefix=''):
"""Takes a list of paths and filters out ignored ones."""
prefix = self._relpath_no_dot(prefix)
prefixed_path_list = [self._append_slash_if_dir_path(os.path.join(prefix, item)) for item in path_list]
ignored_paths = list(self.ignore.match_files(prefixed_path_list))
if len(ignored_paths) == 0:
return path_list
return [fast_relpath(f, prefix).rstrip('/') for f in
[path for path in prefixed_path_list if path not in ignored_paths]
]
def readlink(self, relpath):
link_path = self.relative_readlink(relpath)
if os.path.isabs(link_path):
raise IOError(
"Absolute symlinks not supported in {}: {} -> {}".format(
self, relpath, link_path))
# In order to enforce that this link does not escape the build_root, we join and
# then remove it.
abs_normpath = os.path.normpath(
os.path.join(self.build_root, os.path.dirname(relpath), link_path))
return fast_relpath(abs_normpath, self.build_root)
def render_logs(workdir):
"""Renders all potentially relevant logs from the given workdir to stdout."""
filenames = list(glob.glob(os.path.join(
workdir, "logs/exceptions*log"))) + list(
glob.glob(os.path.join(workdir, "pantsd/pantsd.log")))
for filename in filenames:
rel_filename = fast_relpath(filename, workdir)
print(f"{rel_filename} +++ ")
for line in _read_log(filename):
print(f"{rel_filename} >>> {line}")
print(f"{rel_filename} --- ")
def _eager_fileset_with_spec(spec_path, filespecs, source_files_digest,
excluded_source_files):
excluded = {f.path for f in excluded_source_files.dependencies}
file_tuples = [(fast_relpath(fd.path, spec_path), fd.digest)
for fd in source_files_digest.dependencies
if fd.path not in excluded]
# NB: In order to preserve declared ordering, we record a list of matched files
# independent of the file hash dict.
return EagerFilesetWithSpec(spec_path,
filespecs,
files=tuple(f for f, _ in file_tuples),
file_hashes=dict(file_tuples))
def _create_requirements(self, context, workdir):
tool_subsystem = self._tool_subsystem()
address = Address(spec_path=fast_relpath(workdir, get_buildroot()),
target_name=tool_subsystem.options_scope)
context.build_graph.inject_synthetic_target(
address=address,
target_type=PythonRequirementLibrary,
requirements=[
PythonRequirement(r)
for r in tool_subsystem.get_requirement_specs()
])
return context.build_graph.get_target(address=address)
def _shaded_jar_as_classpath_entry(self, shaded_jar):
# Capture a Snapshot for the jar.
buildroot = get_buildroot()
snapshot = self.context._scheduler.capture_snapshots([
PathGlobsAndRoot(
PathGlobs([fast_relpath(shaded_jar, buildroot)]),
buildroot,
Digest.load(shaded_jar),
)
])[0]
snapshot.digest.dump(shaded_jar)
return ClasspathEntry(shaded_jar, directory_digest=snapshot.digest)
def globs_relative_to_buildroot(self):
buildroot = get_buildroot()
globs = []
for bundle in self.bundles:
fileset = bundle.fileset
if fileset is None:
continue
globs += [fast_relpath(f, buildroot) for f in bundle.filemap.keys()]
super_globs = super().globs_relative_to_buildroot()
if super_globs:
globs += super_globs["globs"]
return {"globs": globs}
def hydrate_bundles(bundles_field, files_content_list):
"""Given a BundlesField and FilesContent for each of its filesets create a list of BundleAdaptors."""
bundles = []
zipped = zip(bundles_field.bundles, bundles_field.filespecs_list, files_content_list)
for bundle, filespecs, files_content in zipped:
spec_path = bundles_field.address.spec_path
file_hashes = {fast_relpath(fc.path, spec_path): sha1(fc.content).digest()
for fc in files_content.dependencies}
kwargs = bundle.kwargs()
kwargs['fileset'] = EagerFilesetWithSpec(spec_path, filespecs, file_hashes)
bundles.append(BundleAdaptor(**kwargs))
return HydratedField('bundles', bundles)
def _double_check_cache_for_vts(self, vts, zinc_compile_context):
# Double check the cache before beginning compilation
if self.check_cache(vts):
self.context.log.debug(f"Snapshotting results for {vts.target.address.spec}")
classpath_entry = self._classpath_for_context(zinc_compile_context)
relpath = fast_relpath(classpath_entry.path, get_buildroot())
(classes_dir_snapshot,) = self.context._scheduler.capture_snapshots(
[PathGlobsAndRoot(PathGlobs([relpath]), get_buildroot(), Digest.load(relpath),),]
)
classpath_entry.hydrate_missing_directory_digest(classes_dir_snapshot.directory_digest)
# Re-validate the vts!
vts.update()
def _compile_hermetic(self, jvm_options, ctx, classes_dir, zinc_args,
compiler_bridge_classpath_entry,
dependency_classpath, scalac_classpath_entries):
zinc_relpath = fast_relpath(self._zinc.zinc, get_buildroot())
snapshots = [
self._zinc.snapshot(self.context._scheduler),
ctx.target.sources_snapshot(self.context._scheduler),
]
relevant_classpath_entries = dependency_classpath + [
compiler_bridge_classpath_entry
]
directory_digests = tuple(entry.directory_digest
for entry in relevant_classpath_entries
if entry.directory_digest)
if len(directory_digests) != len(relevant_classpath_entries):
for dep in relevant_classpath_entries:
if dep.directory_digest is None:
logger.warning(
"ClasspathEntry {} didn't have a Digest, so won't be present for hermetic "
"execution".format(dep))
snapshots.extend(classpath_entry.directory_digest
for classpath_entry in scalac_classpath_entries)
# TODO: Extract something common from Executor._create_command to make the command line
# TODO: Lean on distribution for the bin/java appending here
merged_input_digest = self.context._scheduler.merge_directories(
tuple(s.directory_digest for s in snapshots) + directory_digests)
argv = ['.jdk/bin/java'] + jvm_options + [
'-cp', zinc_relpath, Zinc.ZINC_COMPILE_MAIN
] + zinc_args
req = ExecuteProcessRequest(
argv=tuple(argv),
input_files=merged_input_digest,
output_directories=(classes_dir, ),
description="zinc compile for {}".format(ctx.target.address.spec),
# TODO: These should always be unicodes
# Since this is always hermetic, we need to use `underlying_dist`
jdk_home=text_type(self._zinc.underlying_dist.home),
)
res = self.context.execute_process_synchronously_or_raise(
req, self.name(), [WorkUnitLabel.COMPILER])
# TODO: Materialize as a batch in do_compile or somewhere
self.context._scheduler.materialize_directories(
(DirectoryToMaterialize(get_buildroot(),
res.output_directory_digest), ))
# TODO: This should probably return a ClasspathEntry rather than a Digest
return res.output_directory_digest
def create_requirements(cls, context, workdir):
options = cls.global_instance().get_options()
address = Address(spec_path=fast_relpath(
workdir, get_buildroot()),
target_name='isort')
requirements = ['isort=={}'.format(options.version)
] + options.additional_requirements
context.build_graph.inject_synthetic_target(
address=address,
target_type=PythonRequirementLibrary,
requirements=[PythonRequirement(r) for r in requirements])
return context.build_graph.get_target(address=address)
def snapshot(self, scheduler):
buildroot = get_buildroot()
return scheduler.capture_snapshots((
PathGlobsAndRoot(
PathGlobs(
tuple(
fast_relpath(a, buildroot)
for a in (self.zinc, self.compiler_bridge, self.compiler_interface)
)
),
buildroot,
),
))[0]
def _record_compile_classpath(self, classpath, targets, outdir):
relative_classpaths = [
fast_relpath(path,
self.get_options().pants_workdir)
for path in classpath
]
text = '\n'.join(relative_classpaths)
for target in targets:
path = os.path.join(outdir, 'compile_classpath',
'{}.txt'.format(target.id))
safe_mkdir(os.path.dirname(path), clean=False)
with open(path, 'w') as f:
f.write(text.encode('utf-8'))
def scan_addresses(self, root=None):
if root:
try:
base_path = fast_relpath(root, self._build_root)
except ValueError as e:
raise self.InvalidRootError(e)
else:
base_path = ''
addresses = set()
for address in self._graph.inject_specs_closure([DescendantAddresses(base_path)]):
addresses.add(address)
return addresses
def _eager_fileset_with_spec(spec_path, filespec, snapshot, include_dirs=False):
fds = snapshot.path_stats if include_dirs else snapshot.files
files = tuple(fast_relpath(fd.path, spec_path) for fd in fds)
relpath_adjusted_filespec = FilesetRelPathWrapper.to_filespec(filespec['globs'], spec_path)
if filespec.has_key('exclude'):
relpath_adjusted_filespec['exclude'] = [FilesetRelPathWrapper.to_filespec(e['globs'], spec_path)
for e in filespec['exclude']]
return EagerFilesetWithSpec(spec_path,
relpath_adjusted_filespec,
files=files,
files_hash=snapshot.fingerprint)
def render_logs(workdir):
"""Renders all potentially relevant logs from the given workdir to stdout."""
filenames = list(
glob.glob(os.path.join(workdir, 'logs/exceptions*log'))
) + list(
glob.glob(os.path.join(workdir, 'pantsd/pantsd.log'))
)
for filename in filenames:
rel_filename = fast_relpath(filename, workdir)
print('{} +++ '.format(rel_filename))
for line in _read_log(filename):
print('{} >>> {}'.format(rel_filename, line))
print('{} --- '.format(rel_filename))
def _filemap(self, abs_path):
filemap = OrderedDict()
for path in self.fileset:
if abs_path:
if not os.path.isabs(path):
path = os.path.join(get_buildroot(), self.rel_path, path)
else:
if os.path.isabs(path):
path = fast_relpath(path, get_buildroot())
else:
path = os.path.join(self.rel_path, path)
filemap[path] = self.mapper(path)
return filemap
def scan_build_files(project_tree,
base_relpath,
spec_excludes=None,
build_ignore_patterns=None):
"""Looks for all BUILD files
:param project_tree: Project tree to scan in.
:type project_tree: :class:`pants.base.project_tree.ProjectTree`
:param base_relpath: Directory under root_dir to scan.
:param spec_excludes: List of paths to exclude from the scan. These can be absolute paths
or paths that are relative to the root_dir.
:param build_ignore_patterns: .gitignore like patterns to exclude from BUILD files scan.
:type build_ignore_patterns: pathspec.pathspec.PathSpec
"""
deprecated_conditional(lambda: spec_excludes is not None, '0.0.75',
'Use build_ignore_patterns instead.')
if base_relpath and os.path.isabs(base_relpath):
raise BuildFile.BadPathError(
'base_relpath parameter ({}) should be a relative path.'.
format(base_relpath))
if base_relpath and not project_tree.isdir(base_relpath):
raise BuildFile.BadPathError(
'Can only scan directories and {0} is not a valid dir.'.format(
base_relpath))
if build_ignore_patterns and not isinstance(build_ignore_patterns,
PathSpec):
raise TypeError(
"build_ignore_patterns should be pathspec.pathspec.PathSpec instance, "
"instead {} was given.".format(type(build_ignore_patterns)))
# Hack, will be removed after spec_excludes removal.
build_ignore_patterns = BuildFile._add_spec_excludes_to_build_ignore_patterns(
project_tree.build_root, build_ignore_patterns, spec_excludes)
build_files = set()
for root, dirs, files in project_tree.walk(base_relpath or '',
topdown=True):
excluded_dirs = list(
build_ignore_patterns.match_files(
'{}/'.format(os.path.join(root, dirname))
for dirname in dirs))
for subdir in excluded_dirs:
# Remove trailing '/' from paths which were added to indicate that paths are paths to directories.
dirs.remove(fast_relpath(subdir, root)[:-1])
for filename in files:
if BuildFile._is_buildfile_name(filename):
build_files.add(os.path.join(root, filename))
return BuildFile._build_files_from_paths(project_tree, build_files,
build_ignore_patterns)
def scan_addresses(self, root=None):
if root:
try:
base_path = fast_relpath(root, self._build_root)
except ValueError as e:
raise self.InvalidRootError(e)
else:
base_path = ''
addresses = set()
for address in self._graph.inject_specs_closure(
[DescendantAddresses(base_path)]):
addresses.add(address)
return addresses
def _eager_fileset_with_spec(spec_path, filespec, snapshot):
files = tuple(fast_relpath(fd.path, spec_path) for fd in snapshot.files)
relpath_adjusted_filespec = FilesetRelPathWrapper.to_filespec(filespec['globs'], spec_path)
if filespec.has_key('exclude'):
relpath_adjusted_filespec['exclude'] = [FilesetRelPathWrapper.to_filespec(e['globs'], spec_path)
for e in filespec['exclude']]
# NB: In order to preserve declared ordering, we record a list of matched files
# independent of the file hash dict.
return EagerFilesetWithSpec(spec_path,
relpath_adjusted_filespec,
files=files,
files_hash=snapshot.fingerprint)
def test_v2_list_loop(self):
# Create a BUILD file in a nested temporary directory, and add additional targets to it.
with self.pantsd_test_context(log_level="info") as (
workdir,
config,
checker,
), temporary_dir(root_dir=get_buildroot()) as tmpdir:
rel_tmpdir = fast_relpath(tmpdir, get_buildroot())
def dump(content):
safe_file_dump(os.path.join(tmpdir, "BUILD"), content)
# Dump an initial target before starting the loop.
dump('target(name="one")')
# Launch the loop as a background process.
handle = self.run_pants_with_workdir_without_waiting(
# NB: We disable watchman here because in the context of `--loop`, the total count
# of invalidations matters, and with both `notify` and `watchman` enabled we get
# twice as many.
[
"--no-v1",
"--v2",
"--no-watchman-enable",
"--loop",
"--loop-max=3",
"list",
f"{tmpdir}:",
],
workdir,
config,
)
# Wait for pantsd to come up and for the loop to stabilize.
checker.assert_started()
time.sleep(10)
# Replace the BUILD file content twice.
dump('target(name="two")')
time.sleep(10)
dump('target(name="three")')
# Verify that the three different target states were listed, and that the process exited.
pants_result = handle.join()
self.assert_success(pants_result)
self.assertEquals(
[f"{rel_tmpdir}:{name}" for name in ("one", "two", "three")],
list(pants_result.stdout_data.splitlines()),
)
def _create_context_jar(self, compile_context):
"""Jar up the compile_context to its output jar location.
TODO(stuhood): In the medium term, we hope to add compiler support for this step, which would
allow the jars to be used as compile _inputs_ as well. Currently using jar'd compile outputs as
compile inputs would make the compiler's analysis useless.
see https://github.com/twitter-forks/sbt/tree/stuhood/output-jars
"""
root = compile_context.classes_dir
with compile_context.open_jar(mode='w') as jar:
for abs_sub_dir, dirnames, filenames in safe_walk(root):
for name in dirnames + filenames:
abs_filename = os.path.join(abs_sub_dir, name)
arcname = fast_relpath(abs_filename, root)
jar.write(abs_filename, arcname)
def filter_ignored(self, path_list, prefix=''):
"""Takes a list of paths and filters out ignored ones."""
prefix = self._relpath_no_dot(prefix)
prefixed_path_list = [
self._append_slash_if_dir_path(os.path.join(prefix, item))
for item in path_list
]
ignored_paths = list(self.ignore.match_files(prefixed_path_list))
if len(ignored_paths) == 0:
return path_list
return [
fast_relpath(f, prefix).rstrip('/') for f in
[path for path in prefixed_path_list if path not in ignored_paths]
]
def register_extra_products_from_contexts(self, targets, compile_contexts):
super().register_extra_products_from_contexts(targets, compile_contexts)
def confify(entries):
return [(conf, e) for e in entries for conf in self._confs]
# Ensure that the jar/rsc jar is on the rsc_mixed_compile_classpath.
for target in targets:
merged_cc = compile_contexts[target]
zinc_cc = merged_cc.zinc_cc
rsc_cc = merged_cc.rsc_cc
# Make sure m.jar is digested if it exists when the target is validated.
if rsc_cc.rsc_jar_file.directory_digest is None and os.path.exists(
rsc_cc.rsc_jar_file.path
):
relpath = fast_relpath(rsc_cc.rsc_jar_file.path, get_buildroot())
(classes_dir_snapshot,) = self.context._scheduler.capture_snapshots(
[
PathGlobsAndRoot(
PathGlobs([relpath]), get_buildroot(), Digest.load(relpath),
),
]
)
rsc_cc.rsc_jar_file.hydrate_missing_directory_digest(
classes_dir_snapshot.directory_digest
)
if rsc_cc.workflow is not None:
cp_entries = match(
rsc_cc.workflow,
{
self.JvmCompileWorkflowType.zinc_only: lambda: confify(
[self._classpath_for_context(zinc_cc)]
),
self.JvmCompileWorkflowType.zinc_java: lambda: confify(
[self._classpath_for_context(zinc_cc)]
),
self.JvmCompileWorkflowType.rsc_and_zinc: lambda: confify(
[rsc_cc.rsc_jar_file]
),
self.JvmCompileWorkflowType.outline_and_zinc: lambda: confify(
[rsc_cc.rsc_jar_file]
),
},
)()
self.context.products.get_data("rsc_mixed_compile_classpath").add_for_target(
target, cp_entries
)
def add_directory_digests_for_jars(self, targets_and_jars):
"""For each target, get DirectoryDigests for its jars and return them zipped with the jars.
:param targets_and_jars: List of tuples of the form (Target, [pants.java.jar.jar_dependency_utils.ResolveJar])
:return: list[tuple[(Target, list[pants.java.jar.jar_dependency_utils.ResolveJar])]
"""
targets_and_jars = list(targets_and_jars)
if not targets_and_jars:
return targets_and_jars
jar_paths = []
for target, jars_to_snapshot in targets_and_jars:
for jar in jars_to_snapshot:
jar_paths.append(fast_relpath(jar.pants_path, get_buildroot()))
# Capture Snapshots for jars, using an optional adjacent digest. Create the digest afterward
# if it does not exist.
snapshots = self.context._scheduler.capture_snapshots(
tuple(
PathGlobsAndRoot(
PathGlobs([jar]),
get_buildroot(),
Digest.load(jar),
) for jar in jar_paths))
for snapshot, jar_path in zip(snapshots, jar_paths):
snapshot.digest.dump(jar_path)
# We want to map back the list[Snapshot] to targets_and_jars
# We assume that (1) jars_to_snapshot has the same number of ResolveJars as snapshots does Snapshots,
# and that (2) capture_snapshots preserves ordering.
digests = [snapshot.digest for snapshot in snapshots]
digest_iterator = iter(digests)
snapshotted_targets_and_jars = []
for target, jars_to_snapshot in targets_and_jars:
snapshotted_jars = [
ResolvedJar(
coordinate=jar.coordinate,
cache_path=jar.cache_path,
pants_path=jar.pants_path,
directory_digest=next(digest_iterator),
) for jar in jars_to_snapshot
]
snapshotted_targets_and_jars.append((target, snapshotted_jars))
return snapshotted_targets_and_jars
def _normalize_product_dep(self, buildroot, classes_by_source, dep):
"""Normalizes the given product dep from the given dep into a set of classfiles.
Product deps arrive as sources, jars, and classfiles: this method normalizes them to classfiles.
TODO: This normalization should happen in the super class.
"""
if dep.endswith(".jar"):
# TODO: post sbt/zinc jar output patch, binary deps will be reported directly as classfiles
return set()
elif dep.endswith(".class"):
return set([dep])
else:
# assume a source file and convert to classfiles
rel_src = fast_relpath(dep, buildroot)
return set(p for _, paths in classes_by_source[rel_src].rel_paths() for p in paths)
def _capture_sources(self, vts):
to_capture = []
results_dirs = []
filespecs = []
for vt in vts:
target = vt.target
# Compute the (optional) subdirectory of the results_dir to generate code to. This
# path will end up in the generated FilesetWithSpec and target, and thus needs to be
# located below the stable/symlinked `vt.results_dir`.
synthetic_target_dir = self.synthetic_target_dir(
target, vt.results_dir)
files = self.sources_globs
results_dir_relpath = fast_relpath(synthetic_target_dir,
get_buildroot())
buildroot_relative_globs = tuple(
os.path.join(results_dir_relpath, file) for file in files)
buildroot_relative_excludes = tuple(
os.path.join(results_dir_relpath, file)
for file in self.sources_exclude_globs)
to_capture.append(
PathGlobsAndRoot(
PathGlobs(buildroot_relative_globs,
buildroot_relative_excludes),
text_type(get_buildroot()),
# The digest is stored adjacent to the hash-versioned `vt.current_results_dir`.
Digest.load(vt.current_results_dir),
))
results_dirs.append(results_dir_relpath)
filespecs.append(
FilesetRelPathWrapper.to_filespec(buildroot_relative_globs))
snapshots = self.context._scheduler.capture_snapshots(
tuple(to_capture))
for snapshot, vt in zip(snapshots, vts):
snapshot.directory_digest.dump(vt.current_results_dir)
return tuple(
EagerFilesetWithSpec(
results_dir_relpath,
filespec,
snapshot,
) for (results_dir_relpath, filespec,
snapshot) in zip(results_dirs, filespecs, snapshots))
