def test_analysis_portability(self): # Tests that analysis can be relocated between workdirs and still result in incremental # compile. with temporary_dir() as cache_dir, temporary_dir(root_dir=get_buildroot()) as src_dir, \ temporary_dir(root_dir=get_buildroot(), suffix='.pants.d') as workdir: config = { 'cache.compile.zinc': {'write_to': [cache_dir], 'read_from': [cache_dir]}, } dep_src_file = os.path.join(src_dir, 'org', 'pantsbuild', 'dep', 'A.scala') dep_build_file = os.path.join(src_dir, 'org', 'pantsbuild', 'dep', 'BUILD') con_src_file = os.path.join(src_dir, 'org', 'pantsbuild', 'consumer', 'B.scala') con_build_file = os.path.join(src_dir, 'org', 'pantsbuild', 'consumer', 'BUILD') dep_spec = os.path.join(os.path.basename(src_dir), 'org', 'pantsbuild', 'dep') con_spec = os.path.join(os.path.basename(src_dir), 'org', 'pantsbuild', 'consumer') dep_src = "package org.pantsbuild.dep; class A {}" self.create_file(dep_src_file, dep_src) self.create_file(dep_build_file, "scala_library()") self.create_file(con_src_file, dedent( """package org.pantsbuild.consumer import org.pantsbuild.dep.A class B { def mkA: A = new A() }""")) self.create_file(con_build_file, "scala_library(dependencies=['{}'])".format(dep_spec)) rel_workdir = fast_relpath(workdir, get_buildroot()) rel_src_dir = fast_relpath(src_dir, get_buildroot()) with self.mock_buildroot(dirs_to_copy=[rel_src_dir, rel_workdir]) as buildroot, \ buildroot.pushd(): # 1) Compile in one buildroot. self.run_compile(con_spec, config, os.path.join(buildroot.new_buildroot, rel_workdir)) with self.mock_buildroot(dirs_to_copy=[rel_src_dir, rel_workdir]) as buildroot, \ buildroot.pushd(): # 2) Compile in another buildroot, and check that we hit the cache. new_workdir = os.path.join(buildroot.new_buildroot, rel_workdir) run_two = self.run_compile(con_spec, config, new_workdir) self.assertTrue( re.search( "\[zinc\][^[]*\[cache\][^[]*Using cached artifacts for 2 targets.", run_two.stdout_data), run_two.stdout_data) # 3) Edit the dependency in a way that should trigger an incremental # compile of the consumer. mocked_dep_src_file = os.path.join( buildroot.new_buildroot, fast_relpath(dep_src_file, get_buildroot())) self.create_file(mocked_dep_src_file, dep_src + "; /* this is a comment */") # 4) Compile and confirm that the analysis fetched from the cache in # step 2 causes incrementalism: ie, zinc does not report compiling any files. run_three = self.run_compile(con_spec, config, new_workdir) self.assertTrue( re.search( r"/org/pantsbuild/consumer:consumer\)[^[]*\[compile\][^[]*\[zinc\]\W*\[info\] Compile success", run_three.stdout_data), run_three.stdout_data)
def __init__(self, *args, **kwargs): super(BaseZincCompile, self).__init__(*args, **kwargs) # A directory to contain per-target subdirectories with apt processor info files. self._processor_info_dir = os.path.join(self.workdir, 'apt-processor-info') # Validate zinc options. ZincCompile.validate_arguments(self.context.log, self.get_options().whitelisted_args, self._args) if self.execution_strategy == self.HERMETIC: # TODO: Make incremental compiles work. See: # hermetically https://github.com/pantsbuild/pants/issues/6517 if self.get_options().incremental: raise TaskError("Hermetic zinc execution does not currently support incremental compiles. " "Please use --no-compile-zinc-incremental.") try: fast_relpath(self.get_options().pants_workdir, get_buildroot()) except ValueError: raise TaskError( "Hermetic zinc execution currently requires the workdir to be a child of the buildroot " "but workdir was {} and buildroot was {}".format( self.get_options().pants_workdir, get_buildroot(), ) ) if self.get_options().use_classpath_jars: # TODO: Make this work by capturing the correct Digest and passing them around the # right places. # See https://github.com/pantsbuild/pants/issues/6432 raise TaskError("Hermetic zinc execution currently doesn't work with classpath jars")
def __init__(self, project_tree, relpath, must_exist=True): """Creates a BuildFile object representing the BUILD file family at the specified path. :param project_tree: Project tree the BUILD file exist in. :type project_tree: :class:`pants.base.project_tree.ProjectTree` :param string relpath: The path relative to root_dir where the BUILD file is found - this can either point directly at the BUILD file or else to a directory which contains BUILD files. :param bool must_exist: If True, at least one BUILD file must exist at the given location or else an` `MissingBuildFileError` is thrown :raises IOError: if the root_dir path is not absolute. :raises MissingBuildFileError: if the path does not house a BUILD file and must_exist is `True`. """ self.project_tree = project_tree self.root_dir = project_tree.build_root path = os.path.join(self.root_dir, relpath) if relpath else self.root_dir self._build_basename = self._BUILD_FILE_PREFIX if project_tree.isdir(fast_relpath(path, self.root_dir)): buildfile = os.path.join(path, self._build_basename) else: buildfile = path # There is no BUILD file without a prefix so select any viable sibling buildfile_relpath = fast_relpath(buildfile, self.root_dir) if not project_tree.exists(buildfile_relpath) or project_tree.isdir(buildfile_relpath): for build in self._get_all_build_files(os.path.dirname(buildfile)): self._build_basename = build buildfile = os.path.join(path, self._build_basename) buildfile_relpath = fast_relpath(buildfile, self.root_dir) break if must_exist: if not project_tree.exists(buildfile_relpath): raise self.MissingBuildFileError('BUILD file does not exist at: {path}' .format(path=buildfile)) # If a build file must exist then we want to make sure it's not a dir. # In other cases we are ok with it being a dir, for example someone might have # repo/scripts/build/doit.sh. if project_tree.isdir(buildfile_relpath): raise self.MissingBuildFileError('Path to buildfile ({buildfile}) is a directory, ' 'but it must be a file.'.format(buildfile=buildfile)) if not self._is_buildfile_name(os.path.basename(buildfile)): raise self.MissingBuildFileError('{path} is not a BUILD file' .format(path=buildfile)) self.full_path = os.path.realpath(buildfile) self.name = os.path.basename(self.full_path) self.parent_path = os.path.dirname(self.full_path) self.relpath = fast_relpath(self.full_path, self.root_dir) self.spec_path = os.path.dirname(self.relpath)
def walk(self, relpath, topdown=True): """ Walk the file tree rooted at `path`. Works like os.walk but returned root value is relative path. Ignored paths will not be returned. """ for root, dirs, files in self._walk_raw(relpath, topdown): matched_dirs = self.ignore.match_files([os.path.join(root, "{}/".format(d)) for d in dirs]) matched_files = self.ignore.match_files([os.path.join(root, f) for f in files]) for matched_dir in matched_dirs: dirs.remove(fast_relpath(matched_dir, root).rstrip('/')) for matched_file in matched_files: files.remove(fast_relpath(matched_file, root)) yield root, dirs, files
def scan_addresses(self, root=None): """Recursively gathers all addresses visible under `root` of the virtual address space. :param string root: The absolute path of the root to scan; defaults to the root directory of the pants project. :rtype: set of :class:`pants.build_graph.address.Address` :raises AddressLookupError: if there is a problem parsing a BUILD file """ root_dir = get_buildroot() base_path = None if root: try: base_path = fast_relpath(root, root_dir) except ValueError as e: raise self.InvalidRootError(e) addresses = set() try: for build_file in BuildFile.scan_build_files(self._project_tree, base_relpath=base_path, build_ignore_patterns=self._build_ignore_patterns): for address in self.addresses_in_spec_path(build_file.spec_path): addresses.add(address) except BuildFile.BuildFileError as e: # Handle exception from BuildFile out of paranoia. Currently, there is no way to trigger it. raise self.BuildFileScanError("{message}\n while scanning BUILD files in '{root}'." .format(message=e, root=root)) return addresses
def create_fileset_with_spec(cls, rel_path, *patterns, **kwargs): """ :param rel_path: The relative path to create a FilesetWithSpec for. :param patterns: glob patterns to apply. :param exclude: A list of {,r,z}globs objects, strings, or lists of strings to exclude. """ for pattern in patterns: if not isinstance(pattern, string_types): raise ValueError("Expected string patterns for {}: got {}".format(cls.__name__, patterns)) raw_exclude = kwargs.pop('exclude', []) buildroot = get_buildroot() root = os.path.normpath(os.path.join(buildroot, rel_path)) # making sure there are no unknown arguments. unknown_args = set(kwargs.keys()) - cls.KNOWN_PARAMETERS if unknown_args: raise ValueError('Unexpected arguments while parsing globs: {}'.format( ', '.join(unknown_args))) for glob in patterns: if cls._is_glob_dir_outside_root(glob, root): raise ValueError('Invalid glob {}, points outside BUILD file root {}'.format(glob, root)) exclude = cls.process_raw_exclude(raw_exclude) files_calculator = cls._file_calculator(root, patterns, kwargs, exclude) rel_root = fast_relpath(root, buildroot) if rel_root == '.': rel_root = '' filespec = cls.to_filespec(patterns, root=rel_root, exclude=exclude) return LazyFilesetWithSpec(rel_root, filespec, files_calculator)
def walk(self, relpath, topdown=True): def onerror(error): raise OSError(getattr(error, 'errno', None), 'Failed to walk below {}'.format(relpath), error) for root, dirs, files in safe_walk(os.path.join(self.build_root, relpath), topdown=topdown, onerror=onerror): yield fast_relpath(root, self.build_root), dirs, files
def scan_build_files(project_tree, base_relpath, build_ignore_patterns=None): """Looks for all BUILD files :param project_tree: Project tree to scan in. :type project_tree: :class:`pants.base.project_tree.ProjectTree` :param base_relpath: Directory under root_dir to scan. :param build_ignore_patterns: .gitignore like patterns to exclude from BUILD files scan. :type build_ignore_patterns: pathspec.pathspec.PathSpec """ if base_relpath and os.path.isabs(base_relpath): raise BuildFile.BadPathError('base_relpath parameter ({}) should be a relative path.' .format(base_relpath)) if base_relpath and not project_tree.isdir(base_relpath): raise BuildFile.BadPathError('Can only scan directories and {0} is not a valid dir.' .format(base_relpath)) if build_ignore_patterns and not isinstance(build_ignore_patterns, PathSpec): raise TypeError("build_ignore_patterns should be pathspec.pathspec.PathSpec instance, " "instead {} was given.".format(type(build_ignore_patterns))) build_files = set() for root, dirs, files in project_tree.walk(base_relpath or '', topdown=True): excluded_dirs = list(build_ignore_patterns.match_files('{}/'.format(os.path.join(root, dirname)) for dirname in dirs)) for subdir in excluded_dirs: # Remove trailing '/' from paths which were added to indicate that paths are paths to directories. dirs.remove(fast_relpath(subdir, root)[:-1]) for filename in files: if BuildFile._is_buildfile_name(filename): build_files.add(os.path.join(root, filename)) return BuildFile._build_files_from_paths(project_tree, build_files, build_ignore_patterns)
def _record_compile_classpath(self, classpath, target, outdir): relative_classpaths = [fast_relpath(path, self.get_options().pants_workdir) for path in classpath] text = '\n'.join(relative_classpaths) path = os.path.join(outdir, 'compile_classpath', '{}.txt'.format(target.id)) safe_mkdir(os.path.dirname(path), clean=False) with open(path, 'w') as f: f.write(text)
def _get_all_build_files(self, path): """Returns all the BUILD files on a path""" results = [] relpath = fast_relpath(path, self.root_dir) for build in self.project_tree.glob1(relpath, '{prefix}*'.format(prefix=self._BUILD_FILE_PREFIX)): if self._is_buildfile_name(build) and self.project_tree.isfile(os.path.join(relpath, build)): results.append(build) return sorted(results)
def relativize(paths, project_tree): for path in paths: if os.path.isabs(path): realpath = os.path.realpath(path) if realpath.startswith(project_tree.build_root): yield fast_relpath(realpath, project_tree.build_root) else: yield path
def siblings(self): """Returns an iterator over all the BUILD files co-located with this BUILD file not including this BUILD file itself""" for build in BuildFile.get_build_files_family(self.project_tree, fast_relpath(self.parent_path, self.root_dir)): if self != build: yield build
def _tool_classpath(self, tool, products, scheduler): """Return the proper classpath based on products and scala version.""" classpath = self.tool_classpath_from_products(products, self.versioned_tool_name(tool, self.version), scope=self.options_scope) classpath = tuple(fast_relpath(c, get_buildroot()) for c in classpath) return self._memoized_scalac_classpath(classpath, scheduler)
def _compile_hermetic(self, jvm_options, ctx, classes_dir, zinc_args, compiler_bridge_classpath_entry, dependency_classpath, scalac_classpath_entries): zinc_relpath = fast_relpath(self._zinc.zinc, get_buildroot()) snapshots = [ self._zinc.snapshot(self.context._scheduler), ctx.target.sources_snapshot(self.context._scheduler), ] relevant_classpath_entries = dependency_classpath + [compiler_bridge_classpath_entry] directory_digests = tuple( entry.directory_digest for entry in relevant_classpath_entries if entry.directory_digest ) if len(directory_digests) != len(relevant_classpath_entries): for dep in relevant_classpath_entries: if dep.directory_digest is None: logger.warning( "ClasspathEntry {} didn't have a Digest, so won't be present for hermetic " "execution".format(dep) ) snapshots.extend( classpath_entry.directory_digest for classpath_entry in scalac_classpath_entries ) # TODO: Extract something common from Executor._create_command to make the command line # TODO: Lean on distribution for the bin/java appending here merged_input_digest = self.context._scheduler.merge_directories( tuple(s.directory_digest for s in snapshots) + directory_digests ) argv = ['.jdk/bin/java'] + jvm_options + [ '-cp', zinc_relpath, Zinc.ZINC_COMPILE_MAIN ] + zinc_args # TODO(#6071): Our ExecuteProcessRequest expects a specific string type for arguments, # which py2 doesn't default to. This can be removed when we drop python 2. argv = [text_type(arg) for arg in argv] req = ExecuteProcessRequest( argv=tuple(argv), input_files=merged_input_digest, output_directories=(classes_dir,), description="zinc compile for {}".format(ctx.target.address.spec), # TODO: These should always be unicodes # Since this is always hermetic, we need to use `underlying_dist` jdk_home=text_type(self._zinc.underlying_dist.home), ) res = self.context.execute_process_synchronously_or_raise( req, self.name(), [WorkUnitLabel.COMPILER]) # TODO: Materialize as a batch in do_compile or somewhere self.context._scheduler.materialize_directories(( DirectoryToMaterialize(get_buildroot(), res.output_directory_digest), )) # TODO: This should probably return a ClasspathEntry rather than a Digest return res.output_directory_digest
def _spec_excludes_to_gitignore_syntax(build_root, spec_excludes=None): if spec_excludes: for path in spec_excludes: if os.path.isabs(path): realpath = os.path.realpath(path) if realpath.startswith(build_root): yield '/{}'.format(fast_relpath(realpath, build_root)) else: yield '/{}'.format(path)
def ancestors(self): """Returns all BUILD files in ancestor directories of this BUILD file's parent directory.""" parent_buildfiles = OrderedSet() parentdir = fast_relpath(self.parent_path, self.root_dir) while parentdir != '': parentdir = os.path.dirname(parentdir) parent_buildfiles.update(BuildFile.get_build_files_family(self.project_tree, parentdir)) return parent_buildfiles
def descendants(self, spec_excludes=None): """Returns all BUILD files in descendant directories of this BUILD file's parent directory.""" descendants = self.scan_project_tree_build_files(self.project_tree, fast_relpath(self.parent_path, self.root_dir), spec_excludes=spec_excludes) for sibling in self.family(): descendants.discard(sibling) return descendants
def scan_addresses(self, root=None): if root: try: base_path = fast_relpath(root, self._build_root) except ValueError as e: raise self.InvalidRootError(e) else: base_path = '' return self._internal_scan_specs([DescendantAddresses(base_path)], missing_is_fatal=False)
def hydrate_sources(sources_field, source_files_content, excluded_source_files): """Given a SourcesField and FilesContent for its path_globs, create an EagerFilesetWithSpec.""" excluded = {f.path for f in excluded_source_files.dependencies} spec_path = sources_field.address.spec_path filespecs = sources_field.filespecs print('>>> filespecs are {}; included/excluded are: {} / {}'.format(filespecs, [fc.path for fc in source_files_content.dependencies], excluded)) file_hashes = {fast_relpath(fc.path, spec_path): sha1(fc.content).digest() for fc in source_files_content.dependencies if fc.path not in excluded} return HydratedField('sources', EagerFilesetWithSpec(spec_path, filespecs, file_hashes))
def scan_addresses(self, root=None): if root: try: base_path = fast_relpath(root, self._build_root) except ValueError as e: raise self.InvalidRootError(e) else: base_path = "" return self.scan_specs([DescendantAddresses(base_path)])
def render_logs(workdir): """Renders all potentially relevant logs from the given workdir to stdout.""" filenames = list(glob.glob(os.path.join( workdir, 'logs/exceptions*log'))) + list( glob.glob(os.path.join(workdir, 'pantsd/pantsd.log'))) for filename in filenames: rel_filename = fast_relpath(filename, workdir) print('{} +++ '.format(rel_filename)) for line in _read_log(filename): print('{} >>> {}'.format(rel_filename, line)) print('{} --- '.format(rel_filename))
def _snapshotted_classpath(self, results_dir): relpath = fast_relpath(results_dir, get_buildroot()) (classes_dir_snapshot, ) = self.context._scheduler.capture_snapshots([ PathGlobsAndRoot( PathGlobs([relpath]), get_buildroot(), Digest.load(relpath), ), ]) return ClasspathEntry(results_dir, classes_dir_snapshot.directory_digest)
def _record_compile_classpath(self, classpath, target, outdir): relative_classpaths = [ fast_relpath(path, self.get_options().pants_workdir) for path in classpath ] text = '\n'.join(relative_classpaths) path = os.path.join(outdir, 'compile_classpath', f'{target.id}.txt') safe_mkdir(os.path.dirname(path), clean=False) with open(path, 'w') as f: f.write(text)
def readlink(self, relpath): link_path = self.relative_readlink(relpath) if os.path.isabs(link_path): raise IOError('Absolute symlinks not supported in {}: {} -> {}'.format( self, relpath, link_path)) # In order to enforce that this link does not escape the build_root, we join and # then remove it. abs_normpath = os.path.normpath(os.path.join(self.build_root, os.path.dirname(relpath), link_path)) return fast_relpath(abs_normpath, self.build_root)
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # A directory to contain per-target subdirectories with apt processor info files. self._processor_info_dir = os.path.join(self.workdir, 'apt-processor-info') # Validate zinc options. ZincCompile.validate_arguments(self.context.log, self.get_options().whitelisted_args, self._args) if self.execution_strategy == self.ExecutionStrategy.hermetic: try: fast_relpath(self.get_options().pants_workdir, get_buildroot()) except ValueError: raise TaskError( "Hermetic zinc execution currently requires the workdir to be a child of the buildroot " "but workdir was {} and buildroot was {}".format( self.get_options().pants_workdir, get_buildroot(), ))
def _eager_fileset_with_spec(spec_path, filespecs, source_files_digest, excluded_source_files): excluded = {f.path for f in excluded_source_files.dependencies} file_tuples = [(fast_relpath(fd.path, spec_path), fd.digest) for fd in source_files_digest.dependencies if fd.path not in excluded] # NB: In order to preserve declared ordering, we record a list of matched files # independent of the file hash dict. return EagerFilesetWithSpec(spec_path, filespecs, files=tuple(f for f, _ in file_tuples), file_hashes=dict(file_tuples))
def descendants(self, spec_excludes=None): """Returns all BUILD files in descendant directories of this BUILD file's parent directory.""" descendants = BuildFile.scan_build_files(self.project_tree, fast_relpath( self.parent_path, self.root_dir), spec_excludes=spec_excludes) for sibling in self.family(): descendants.discard(sibling) return descendants
def filter_ignored(self, path_list, prefix=''): """Takes a list of paths and filters out ignored ones.""" prefix = self._relpath_no_dot(prefix) prefixed_path_list = [self._append_slash_if_dir_path(os.path.join(prefix, item)) for item in path_list] ignored_paths = list(self.ignore.match_files(prefixed_path_list)) if len(ignored_paths) == 0: return path_list return [fast_relpath(f, prefix).rstrip('/') for f in [path for path in prefixed_path_list if path not in ignored_paths] ]
def readlink(self, relpath): link_path = self.relative_readlink(relpath) if os.path.isabs(link_path): raise IOError( "Absolute symlinks not supported in {}: {} -> {}".format( self, relpath, link_path)) # In order to enforce that this link does not escape the build_root, we join and # then remove it. abs_normpath = os.path.normpath( os.path.join(self.build_root, os.path.dirname(relpath), link_path)) return fast_relpath(abs_normpath, self.build_root)
def render_logs(workdir): """Renders all potentially relevant logs from the given workdir to stdout.""" filenames = list(glob.glob(os.path.join( workdir, "logs/exceptions*log"))) + list( glob.glob(os.path.join(workdir, "pantsd/pantsd.log"))) for filename in filenames: rel_filename = fast_relpath(filename, workdir) print(f"{rel_filename} +++ ") for line in _read_log(filename): print(f"{rel_filename} >>> {line}") print(f"{rel_filename} --- ")
def _create_requirements(self, context, workdir): tool_subsystem = self._tool_subsystem() address = Address(spec_path=fast_relpath(workdir, get_buildroot()), target_name=tool_subsystem.options_scope) context.build_graph.inject_synthetic_target( address=address, target_type=PythonRequirementLibrary, requirements=[ PythonRequirement(r) for r in tool_subsystem.get_requirement_specs() ]) return context.build_graph.get_target(address=address)
def _shaded_jar_as_classpath_entry(self, shaded_jar): # Capture a Snapshot for the jar. buildroot = get_buildroot() snapshot = self.context._scheduler.capture_snapshots([ PathGlobsAndRoot( PathGlobs([fast_relpath(shaded_jar, buildroot)]), buildroot, Digest.load(shaded_jar), ) ])[0] snapshot.digest.dump(shaded_jar) return ClasspathEntry(shaded_jar, directory_digest=snapshot.digest)
def globs_relative_to_buildroot(self): buildroot = get_buildroot() globs = [] for bundle in self.bundles: fileset = bundle.fileset if fileset is None: continue globs += [fast_relpath(f, buildroot) for f in bundle.filemap.keys()] super_globs = super().globs_relative_to_buildroot() if super_globs: globs += super_globs["globs"] return {"globs": globs}
def hydrate_bundles(bundles_field, files_content_list): """Given a BundlesField and FilesContent for each of its filesets create a list of BundleAdaptors.""" bundles = [] zipped = zip(bundles_field.bundles, bundles_field.filespecs_list, files_content_list) for bundle, filespecs, files_content in zipped: spec_path = bundles_field.address.spec_path file_hashes = {fast_relpath(fc.path, spec_path): sha1(fc.content).digest() for fc in files_content.dependencies} kwargs = bundle.kwargs() kwargs['fileset'] = EagerFilesetWithSpec(spec_path, filespecs, file_hashes) bundles.append(BundleAdaptor(**kwargs)) return HydratedField('bundles', bundles)
def _double_check_cache_for_vts(self, vts, zinc_compile_context): # Double check the cache before beginning compilation if self.check_cache(vts): self.context.log.debug(f"Snapshotting results for {vts.target.address.spec}") classpath_entry = self._classpath_for_context(zinc_compile_context) relpath = fast_relpath(classpath_entry.path, get_buildroot()) (classes_dir_snapshot,) = self.context._scheduler.capture_snapshots( [PathGlobsAndRoot(PathGlobs([relpath]), get_buildroot(), Digest.load(relpath),),] ) classpath_entry.hydrate_missing_directory_digest(classes_dir_snapshot.directory_digest) # Re-validate the vts! vts.update()
def _compile_hermetic(self, jvm_options, ctx, classes_dir, zinc_args, compiler_bridge_classpath_entry, dependency_classpath, scalac_classpath_entries): zinc_relpath = fast_relpath(self._zinc.zinc, get_buildroot()) snapshots = [ self._zinc.snapshot(self.context._scheduler), ctx.target.sources_snapshot(self.context._scheduler), ] relevant_classpath_entries = dependency_classpath + [ compiler_bridge_classpath_entry ] directory_digests = tuple(entry.directory_digest for entry in relevant_classpath_entries if entry.directory_digest) if len(directory_digests) != len(relevant_classpath_entries): for dep in relevant_classpath_entries: if dep.directory_digest is None: logger.warning( "ClasspathEntry {} didn't have a Digest, so won't be present for hermetic " "execution".format(dep)) snapshots.extend(classpath_entry.directory_digest for classpath_entry in scalac_classpath_entries) # TODO: Extract something common from Executor._create_command to make the command line # TODO: Lean on distribution for the bin/java appending here merged_input_digest = self.context._scheduler.merge_directories( tuple(s.directory_digest for s in snapshots) + directory_digests) argv = ['.jdk/bin/java'] + jvm_options + [ '-cp', zinc_relpath, Zinc.ZINC_COMPILE_MAIN ] + zinc_args req = ExecuteProcessRequest( argv=tuple(argv), input_files=merged_input_digest, output_directories=(classes_dir, ), description="zinc compile for {}".format(ctx.target.address.spec), # TODO: These should always be unicodes # Since this is always hermetic, we need to use `underlying_dist` jdk_home=text_type(self._zinc.underlying_dist.home), ) res = self.context.execute_process_synchronously_or_raise( req, self.name(), [WorkUnitLabel.COMPILER]) # TODO: Materialize as a batch in do_compile or somewhere self.context._scheduler.materialize_directories( (DirectoryToMaterialize(get_buildroot(), res.output_directory_digest), )) # TODO: This should probably return a ClasspathEntry rather than a Digest return res.output_directory_digest
def create_requirements(cls, context, workdir): options = cls.global_instance().get_options() address = Address(spec_path=fast_relpath( workdir, get_buildroot()), target_name='isort') requirements = ['isort=={}'.format(options.version) ] + options.additional_requirements context.build_graph.inject_synthetic_target( address=address, target_type=PythonRequirementLibrary, requirements=[PythonRequirement(r) for r in requirements]) return context.build_graph.get_target(address=address)
def snapshot(self, scheduler): buildroot = get_buildroot() return scheduler.capture_snapshots(( PathGlobsAndRoot( PathGlobs( tuple( fast_relpath(a, buildroot) for a in (self.zinc, self.compiler_bridge, self.compiler_interface) ) ), buildroot, ), ))[0]
def _record_compile_classpath(self, classpath, targets, outdir): relative_classpaths = [ fast_relpath(path, self.get_options().pants_workdir) for path in classpath ] text = '\n'.join(relative_classpaths) for target in targets: path = os.path.join(outdir, 'compile_classpath', '{}.txt'.format(target.id)) safe_mkdir(os.path.dirname(path), clean=False) with open(path, 'w') as f: f.write(text.encode('utf-8'))
def scan_addresses(self, root=None): if root: try: base_path = fast_relpath(root, self._build_root) except ValueError as e: raise self.InvalidRootError(e) else: base_path = '' addresses = set() for address in self._graph.inject_specs_closure([DescendantAddresses(base_path)]): addresses.add(address) return addresses
def _eager_fileset_with_spec(spec_path, filespec, snapshot, include_dirs=False): fds = snapshot.path_stats if include_dirs else snapshot.files files = tuple(fast_relpath(fd.path, spec_path) for fd in fds) relpath_adjusted_filespec = FilesetRelPathWrapper.to_filespec(filespec['globs'], spec_path) if filespec.has_key('exclude'): relpath_adjusted_filespec['exclude'] = [FilesetRelPathWrapper.to_filespec(e['globs'], spec_path) for e in filespec['exclude']] return EagerFilesetWithSpec(spec_path, relpath_adjusted_filespec, files=files, files_hash=snapshot.fingerprint)
def render_logs(workdir): """Renders all potentially relevant logs from the given workdir to stdout.""" filenames = list( glob.glob(os.path.join(workdir, 'logs/exceptions*log')) ) + list( glob.glob(os.path.join(workdir, 'pantsd/pantsd.log')) ) for filename in filenames: rel_filename = fast_relpath(filename, workdir) print('{} +++ '.format(rel_filename)) for line in _read_log(filename): print('{} >>> {}'.format(rel_filename, line)) print('{} --- '.format(rel_filename))
def _filemap(self, abs_path): filemap = OrderedDict() for path in self.fileset: if abs_path: if not os.path.isabs(path): path = os.path.join(get_buildroot(), self.rel_path, path) else: if os.path.isabs(path): path = fast_relpath(path, get_buildroot()) else: path = os.path.join(self.rel_path, path) filemap[path] = self.mapper(path) return filemap
def scan_build_files(project_tree, base_relpath, spec_excludes=None, build_ignore_patterns=None): """Looks for all BUILD files :param project_tree: Project tree to scan in. :type project_tree: :class:`pants.base.project_tree.ProjectTree` :param base_relpath: Directory under root_dir to scan. :param spec_excludes: List of paths to exclude from the scan. These can be absolute paths or paths that are relative to the root_dir. :param build_ignore_patterns: .gitignore like patterns to exclude from BUILD files scan. :type build_ignore_patterns: pathspec.pathspec.PathSpec """ deprecated_conditional(lambda: spec_excludes is not None, '0.0.75', 'Use build_ignore_patterns instead.') if base_relpath and os.path.isabs(base_relpath): raise BuildFile.BadPathError( 'base_relpath parameter ({}) should be a relative path.'. format(base_relpath)) if base_relpath and not project_tree.isdir(base_relpath): raise BuildFile.BadPathError( 'Can only scan directories and {0} is not a valid dir.'.format( base_relpath)) if build_ignore_patterns and not isinstance(build_ignore_patterns, PathSpec): raise TypeError( "build_ignore_patterns should be pathspec.pathspec.PathSpec instance, " "instead {} was given.".format(type(build_ignore_patterns))) # Hack, will be removed after spec_excludes removal. build_ignore_patterns = BuildFile._add_spec_excludes_to_build_ignore_patterns( project_tree.build_root, build_ignore_patterns, spec_excludes) build_files = set() for root, dirs, files in project_tree.walk(base_relpath or '', topdown=True): excluded_dirs = list( build_ignore_patterns.match_files( '{}/'.format(os.path.join(root, dirname)) for dirname in dirs)) for subdir in excluded_dirs: # Remove trailing '/' from paths which were added to indicate that paths are paths to directories. dirs.remove(fast_relpath(subdir, root)[:-1]) for filename in files: if BuildFile._is_buildfile_name(filename): build_files.add(os.path.join(root, filename)) return BuildFile._build_files_from_paths(project_tree, build_files, build_ignore_patterns)
def scan_addresses(self, root=None): if root: try: base_path = fast_relpath(root, self._build_root) except ValueError as e: raise self.InvalidRootError(e) else: base_path = '' addresses = set() for address in self._graph.inject_specs_closure( [DescendantAddresses(base_path)]): addresses.add(address) return addresses
def _eager_fileset_with_spec(spec_path, filespec, snapshot): files = tuple(fast_relpath(fd.path, spec_path) for fd in snapshot.files) relpath_adjusted_filespec = FilesetRelPathWrapper.to_filespec(filespec['globs'], spec_path) if filespec.has_key('exclude'): relpath_adjusted_filespec['exclude'] = [FilesetRelPathWrapper.to_filespec(e['globs'], spec_path) for e in filespec['exclude']] # NB: In order to preserve declared ordering, we record a list of matched files # independent of the file hash dict. return EagerFilesetWithSpec(spec_path, relpath_adjusted_filespec, files=files, files_hash=snapshot.fingerprint)
def test_v2_list_loop(self): # Create a BUILD file in a nested temporary directory, and add additional targets to it. with self.pantsd_test_context(log_level="info") as ( workdir, config, checker, ), temporary_dir(root_dir=get_buildroot()) as tmpdir: rel_tmpdir = fast_relpath(tmpdir, get_buildroot()) def dump(content): safe_file_dump(os.path.join(tmpdir, "BUILD"), content) # Dump an initial target before starting the loop. dump('target(name="one")') # Launch the loop as a background process. handle = self.run_pants_with_workdir_without_waiting( # NB: We disable watchman here because in the context of `--loop`, the total count # of invalidations matters, and with both `notify` and `watchman` enabled we get # twice as many. [ "--no-v1", "--v2", "--no-watchman-enable", "--loop", "--loop-max=3", "list", f"{tmpdir}:", ], workdir, config, ) # Wait for pantsd to come up and for the loop to stabilize. checker.assert_started() time.sleep(10) # Replace the BUILD file content twice. dump('target(name="two")') time.sleep(10) dump('target(name="three")') # Verify that the three different target states were listed, and that the process exited. pants_result = handle.join() self.assert_success(pants_result) self.assertEquals( [f"{rel_tmpdir}:{name}" for name in ("one", "two", "three")], list(pants_result.stdout_data.splitlines()), )
def _create_context_jar(self, compile_context): """Jar up the compile_context to its output jar location. TODO(stuhood): In the medium term, we hope to add compiler support for this step, which would allow the jars to be used as compile _inputs_ as well. Currently using jar'd compile outputs as compile inputs would make the compiler's analysis useless. see https://github.com/twitter-forks/sbt/tree/stuhood/output-jars """ root = compile_context.classes_dir with compile_context.open_jar(mode='w') as jar: for abs_sub_dir, dirnames, filenames in safe_walk(root): for name in dirnames + filenames: abs_filename = os.path.join(abs_sub_dir, name) arcname = fast_relpath(abs_filename, root) jar.write(abs_filename, arcname)
def filter_ignored(self, path_list, prefix=''): """Takes a list of paths and filters out ignored ones.""" prefix = self._relpath_no_dot(prefix) prefixed_path_list = [ self._append_slash_if_dir_path(os.path.join(prefix, item)) for item in path_list ] ignored_paths = list(self.ignore.match_files(prefixed_path_list)) if len(ignored_paths) == 0: return path_list return [ fast_relpath(f, prefix).rstrip('/') for f in [path for path in prefixed_path_list if path not in ignored_paths] ]
def register_extra_products_from_contexts(self, targets, compile_contexts): super().register_extra_products_from_contexts(targets, compile_contexts) def confify(entries): return [(conf, e) for e in entries for conf in self._confs] # Ensure that the jar/rsc jar is on the rsc_mixed_compile_classpath. for target in targets: merged_cc = compile_contexts[target] zinc_cc = merged_cc.zinc_cc rsc_cc = merged_cc.rsc_cc # Make sure m.jar is digested if it exists when the target is validated. if rsc_cc.rsc_jar_file.directory_digest is None and os.path.exists( rsc_cc.rsc_jar_file.path ): relpath = fast_relpath(rsc_cc.rsc_jar_file.path, get_buildroot()) (classes_dir_snapshot,) = self.context._scheduler.capture_snapshots( [ PathGlobsAndRoot( PathGlobs([relpath]), get_buildroot(), Digest.load(relpath), ), ] ) rsc_cc.rsc_jar_file.hydrate_missing_directory_digest( classes_dir_snapshot.directory_digest ) if rsc_cc.workflow is not None: cp_entries = match( rsc_cc.workflow, { self.JvmCompileWorkflowType.zinc_only: lambda: confify( [self._classpath_for_context(zinc_cc)] ), self.JvmCompileWorkflowType.zinc_java: lambda: confify( [self._classpath_for_context(zinc_cc)] ), self.JvmCompileWorkflowType.rsc_and_zinc: lambda: confify( [rsc_cc.rsc_jar_file] ), self.JvmCompileWorkflowType.outline_and_zinc: lambda: confify( [rsc_cc.rsc_jar_file] ), }, )() self.context.products.get_data("rsc_mixed_compile_classpath").add_for_target( target, cp_entries )
def add_directory_digests_for_jars(self, targets_and_jars): """For each target, get DirectoryDigests for its jars and return them zipped with the jars. :param targets_and_jars: List of tuples of the form (Target, [pants.java.jar.jar_dependency_utils.ResolveJar]) :return: list[tuple[(Target, list[pants.java.jar.jar_dependency_utils.ResolveJar])] """ targets_and_jars = list(targets_and_jars) if not targets_and_jars: return targets_and_jars jar_paths = [] for target, jars_to_snapshot in targets_and_jars: for jar in jars_to_snapshot: jar_paths.append(fast_relpath(jar.pants_path, get_buildroot())) # Capture Snapshots for jars, using an optional adjacent digest. Create the digest afterward # if it does not exist. snapshots = self.context._scheduler.capture_snapshots( tuple( PathGlobsAndRoot( PathGlobs([jar]), get_buildroot(), Digest.load(jar), ) for jar in jar_paths)) for snapshot, jar_path in zip(snapshots, jar_paths): snapshot.digest.dump(jar_path) # We want to map back the list[Snapshot] to targets_and_jars # We assume that (1) jars_to_snapshot has the same number of ResolveJars as snapshots does Snapshots, # and that (2) capture_snapshots preserves ordering. digests = [snapshot.digest for snapshot in snapshots] digest_iterator = iter(digests) snapshotted_targets_and_jars = [] for target, jars_to_snapshot in targets_and_jars: snapshotted_jars = [ ResolvedJar( coordinate=jar.coordinate, cache_path=jar.cache_path, pants_path=jar.pants_path, directory_digest=next(digest_iterator), ) for jar in jars_to_snapshot ] snapshotted_targets_and_jars.append((target, snapshotted_jars)) return snapshotted_targets_and_jars
def _normalize_product_dep(self, buildroot, classes_by_source, dep): """Normalizes the given product dep from the given dep into a set of classfiles. Product deps arrive as sources, jars, and classfiles: this method normalizes them to classfiles. TODO: This normalization should happen in the super class. """ if dep.endswith(".jar"): # TODO: post sbt/zinc jar output patch, binary deps will be reported directly as classfiles return set() elif dep.endswith(".class"): return set([dep]) else: # assume a source file and convert to classfiles rel_src = fast_relpath(dep, buildroot) return set(p for _, paths in classes_by_source[rel_src].rel_paths() for p in paths)
def _capture_sources(self, vts): to_capture = [] results_dirs = [] filespecs = [] for vt in vts: target = vt.target # Compute the (optional) subdirectory of the results_dir to generate code to. This # path will end up in the generated FilesetWithSpec and target, and thus needs to be # located below the stable/symlinked `vt.results_dir`. synthetic_target_dir = self.synthetic_target_dir( target, vt.results_dir) files = self.sources_globs results_dir_relpath = fast_relpath(synthetic_target_dir, get_buildroot()) buildroot_relative_globs = tuple( os.path.join(results_dir_relpath, file) for file in files) buildroot_relative_excludes = tuple( os.path.join(results_dir_relpath, file) for file in self.sources_exclude_globs) to_capture.append( PathGlobsAndRoot( PathGlobs(buildroot_relative_globs, buildroot_relative_excludes), text_type(get_buildroot()), # The digest is stored adjacent to the hash-versioned `vt.current_results_dir`. Digest.load(vt.current_results_dir), )) results_dirs.append(results_dir_relpath) filespecs.append( FilesetRelPathWrapper.to_filespec(buildroot_relative_globs)) snapshots = self.context._scheduler.capture_snapshots( tuple(to_capture)) for snapshot, vt in zip(snapshots, vts): snapshot.directory_digest.dump(vt.current_results_dir) return tuple( EagerFilesetWithSpec( results_dir_relpath, filespec, snapshot, ) for (results_dir_relpath, filespec, snapshot) in zip(results_dirs, filespecs, snapshots))