def _handle_duplicate_sources(self, vt, sources):
"""Handles duplicate sources generated by the given gen target by either failure or deletion.
This method should be called after all dependencies have been injected into the graph, but
before injecting the synthetic version of this target.
Returns a boolean indicating whether it modified the underlying filesystem.
NB(gm): Some code generators may re-generate code that their dependent libraries generate.
This results in targets claiming to generate sources that they really don't, so we try to
filter out sources that were actually generated by dependencies of the target. This causes
the code generated by the dependencies to 'win' over the code generated by dependees. By
default, this behavior is disabled, and duplication in generated sources will raise a
TaskError. This is controlled by the --allow-dups flag.
"""
target = vt.target
target_workdir = vt.results_dir
# Walk dependency gentargets and record any sources owned by those targets that are also
# owned by this target.
duplicates_by_target = OrderedDict()
def record_duplicates(dep):
if dep == target or not self.is_gentarget(dep.concrete_derived_from):
return False
duped_sources = [s for s in dep.sources_relative_to_source_root() if s in sources.files and
not self.ignore_dup(target, dep, s)]
if duped_sources:
duplicates_by_target[dep] = duped_sources
target.walk(record_duplicates)
# If there were no dupes, we're done.
if not duplicates_by_target:
return False
# If there were duplicates warn or error.
messages = ['{target} generated sources that had already been generated by dependencies.'
.format(target=target.address.spec)]
for dep, duped_sources in duplicates_by_target.items():
messages.append('\t{} also generated:'.format(dep.concrete_derived_from.address.spec))
messages.extend(['\t\t{}'.format(source) for source in duped_sources])
message = '\n'.join(messages)
if self.get_options().allow_dups:
logger.warn(message)
else:
raise self.DuplicateSourceError(message)
did_modify = False
# Finally, remove duplicates from the workdir. This prevents us from having to worry
# about them during future incremental compiles.
for dep, duped_sources in duplicates_by_target.items():
for duped_source in duped_sources:
safe_delete(os.path.join(target_workdir, duped_source))
did_modify = True
if did_modify:
Digest.clear(vt.current_results_dir)
return did_modify
def __init__(self, info_file):
self._info_file = info_file
safe_mkdir_for(self._info_file)
self._info = OrderedDict()
if os.path.exists(self._info_file):
with open(self._info_file, 'r') as infile:
info = infile.read()
for m in re.finditer("""^([^:]+):(.*)$""", info, re.MULTILINE):
self._info[m.group(1).strip()] = m.group(2).strip()
def reset(self):
"""Clear out the state of the BuildGraph, in particular Target mappings and dependencies.
:API: public
"""
self._target_by_address = OrderedDict()
self._target_dependencies_by_address = defaultdict(OrderedSet)
self._target_dependees_by_address = defaultdict(OrderedSet)
self._derived_from_by_derivative = {} # Address -> Address.
self._derivatives_by_derived_from = defaultdict(list) # Address -> list of Address.
self.synthetic_addresses = set()
def execute(self):
codegen_targets = self.codegen_targets()
if not codegen_targets:
return
self._validate_sources_globs()
with self.invalidated(
codegen_targets,
invalidate_dependents=True,
topological_order=True,
fingerprint_strategy=self.get_fingerprint_strategy(
)) as invalidation_check:
with self.context.new_workunit(name='execute',
labels=[WorkUnitLabel.MULTITOOL]):
vts_to_sources = OrderedDict()
for vt in invalidation_check.all_vts:
synthetic_target_dir = self.synthetic_target_dir(
vt.target, vt.results_dir)
key = (vt, synthetic_target_dir)
vts_to_sources[key] = None
# Build the target and handle duplicate sources.
if not vt.valid:
if self._do_validate_sources_present(vt.target):
self.execute_codegen(vt.target, vt.results_dir)
sources = self._capture_sources((key, ))[0]
# _handle_duplicate_sources may delete files from the filesystem, so we need to
# re-capture the sources.
if not self._handle_duplicate_sources(
vt.target, vt.results_dir, sources):
vts_to_sources[key] = sources
vt.update()
vts_to_capture = tuple(
key for key, sources in vts_to_sources.items()
if sources is None)
filesets = self._capture_sources(vts_to_capture)
for key, fileset in zip(vts_to_capture, filesets):
vts_to_sources[key] = fileset
for (vt,
synthetic_target_dir), fileset in vts_to_sources.items():
self._inject_synthetic_target(
vt.target,
synthetic_target_dir,
fileset,
)
self._mark_transitive_invalidation_hashes_dirty(
vt.target.address for vt in invalidation_check.all_vts)
def create(cls, rule_entries, union_rules=None):
"""Creates a RuleIndex with tasks indexed by their output type."""
serializable_rules = OrderedDict()
serializable_roots = OrderedSet()
union_rules = OrderedDict(union_rules or ())
def add_task(product_type, rule):
# TODO(#7311): make a defaultdict-like wrapper for OrderedDict if more widely used.
if product_type not in serializable_rules:
serializable_rules[product_type] = OrderedSet()
serializable_rules[product_type].add(rule)
def add_root_rule(root_rule):
serializable_roots.add(root_rule)
def add_rule(rule):
if isinstance(rule, RootRule):
add_root_rule(rule)
else:
add_task(rule.output_type, rule)
for dep_rule in rule.dependency_rules:
add_rule(dep_rule)
def add_type_transition_rule(union_rule):
# NB: This does not require that union bases be supplied to `def rules():`, as the union type
# is never instantiated!
union_base = union_rule.union_base
assert union_base._is_union
union_member = union_rule.union_member
if union_base not in union_rules:
union_rules[union_base] = OrderedSet()
union_rules[union_base].add(union_member)
for entry in rule_entries:
if isinstance(entry, Rule):
add_rule(entry)
elif isinstance(entry, UnionRule):
add_type_transition_rule(entry)
elif hasattr(entry, '__call__'):
rule = getattr(entry, 'rule', None)
if rule is None:
raise TypeError(
"Expected callable {} to be decorated with @rule.".
format(entry))
add_rule(rule)
else:
raise TypeError("""\
Rule entry {} had an unexpected type: {}. Rules either extend Rule or UnionRule, or are static \
functions decorated with @rule.""".format(entry, type(entry)))
return cls(serializable_rules, serializable_roots, union_rules)
def default(self, o):
if isinstance(o, Mapping):
# Preserve order to avoid collisions for OrderedDict inputs to json.dumps(). We don't do this
# for general mappings because dicts have an arbitrary key ordering in some versions of python
# 3 (2.7 and 3.6-3.7 are known to have sorted keys, but with different definitions of sorted
# orders across versions, including insertion order). We want unordered dicts to collide if
# they have the same keys, in the same way we special-case sets below. Calling sorted() should
# be very fast if the keys happen to be pre-sorted. Pants options don't support OrderedDict
# inputs, and allowing them creates an ambiguity we don't need to deal with right now. See
# discussion in #6475.
if isinstance(o, OrderedDict):
raise TypeError(
'{cls} does not support OrderedDict inputs: {val!r}.'.
format(cls=type(self).__name__, val=o))
# TODO(#7082): we can remove the sorted() and OrderedDict when we drop python 2.7 and simply
# ensure we encode the keys/values as we do right here.
ordered_kv_pairs = sorted(o.items(), key=lambda x: x[0])
return OrderedDict(
(self._maybe_encode_dict_key(k), self.default(v))
for k, v in ordered_kv_pairs)
elif isinstance(o, Set):
# We disallow OrderedSet (although it is not a stdlib collection) for the same reasons as
# OrderedDict above.
if isinstance(o, OrderedSet):
raise TypeError(
'{cls} does not support OrderedSet inputs: {val!r}.'.
format(cls=type(self).__name__, val=o))
# Set order is arbitrary in python 3.6 and 3.7, so we need to keep this sorted() call.
return sorted(self.default(i) for i in o)
elif isinstance(o, Iterable) and not isinstance(o, (bytes, list, str)):
return list(self.default(i) for i in o)
return o
def _singletons(cls):
"""Generate memoized instances of this enum wrapping each of this enum's allowed values.
NB: The implementation of enum() should use this property as the source of truth for allowed
values and enum instances from those values.
"""
return OrderedDict((value, cls._make_singleton(value)) for value in all_values_realized)
def test_classpath_by_targets(self):
b = self.make_target('b', JvmTarget)
a = self.make_target('a',
JvmTarget,
dependencies=[b],
excludes=[Exclude('com.example', 'lib')])
classpath_products = ClasspathProducts(self.pants_workdir)
path1 = self._path('jar/path1')
path2 = self._path('jar/path2')
path3 = os.path.join(self.pants_workdir, 'jar/path3')
resolved_jar = ResolvedJar(M2Coordinate(org='com.example',
name='lib',
rev='1.0'),
cache_path='somewhere',
pants_path=path3)
classpath_products.add_for_target(a, [('default', path1)])
classpath_products.add_for_target(a, [('non-default', path2)])
classpath_products.add_for_target(b, [('default', path2)])
classpath_products.add_jars_for_targets([b], 'default', [resolved_jar])
classpath_products.add_excludes_for_targets([a])
# (a, path2) filtered because of conf
# (b, path3) filtered because of excludes
self.assertEqual(
OrderedDict([(a, [ClasspathEntry(path1)]),
(b, [ClasspathEntry(path2)])]),
ClasspathUtil.classpath_by_targets(a.closure(bfs=True),
classpath_products))
def classpath_by_targets(cls,
targets,
classpath_products,
confs=('default', )):
"""Return classpath entries grouped by their targets for the given `targets`.
:param targets: The targets to lookup classpath products for.
:param ClasspathProducts classpath_products: Product containing classpath elements.
:param confs: The list of confs for use by this classpath.
:returns: The ordered (target, classpath) mappings.
:rtype: OrderedDict
"""
classpath_target_tuples = classpath_products.get_product_target_mappings_for_targets(
targets)
filtered_items_iter = filter(
cls._accept_conf_filter(confs, lambda x: x[0][0]),
classpath_target_tuples)
# group (classpath_entry, target) tuples by targets
target_to_classpath = OrderedDict()
for classpath_entry, target in filtered_items_iter:
_, entry = classpath_entry
if not target in target_to_classpath:
target_to_classpath[target] = []
target_to_classpath[target].append(entry)
return target_to_classpath
def test_execution_reduced_dependencies_1(self):
dep_map = OrderedDict(foo=['bar'], bar=['baz'], baz=[])
target_map = self.create_dependencies(dep_map)
with self.run_execute(target_map['foo'], recursive=False) as created:
self.assertEqual([target_map['foo']], list(created.keys()))
with self.run_execute(target_map['foo'], recursive=True) as created:
self.assertEqual(
{target_map['baz'], target_map['bar'], target_map['foo']},
set(created.keys()))
def _visit_goal(self, goal, context, goal_info_by_goal):
if goal in goal_info_by_goal:
return
tasktypes_by_name = OrderedDict()
goal_dependencies = set()
visited_task_types = set()
for task_name in reversed(goal.ordered_task_names()):
task_type = goal.task_type_by_name(task_name)
tasktypes_by_name[task_name] = task_type
visited_task_types.add(task_type)
round_manager = RoundManager(context)
task_type.invoke_prepare(context.options, round_manager)
try:
dependencies = round_manager.get_dependencies()
for producer_info in dependencies:
producer_goal = producer_info.goal
if producer_goal == goal:
if producer_info.task_type == task_type:
# We allow a task to produce products it itself needs. We trust the Task writer
# to arrange for proper sequencing.
pass
elif producer_info.task_type in visited_task_types:
ordering = '\n\t'.join(
"[{0}] '{1}' {2}".format(
i, tn,
goal.task_type_by_name(tn).__name__)
for i, tn in enumerate(
goal.ordered_task_names()))
raise self.TaskOrderError(
"TaskRegistrar '{name}' with action {consumer_task} depends on {data} from task "
"{producer_task} which is ordered after it in the '{goal}' goal:\n\t{ordering}"
.format(name=task_name,
consumer_task=task_type.__name__,
data=producer_info.product_type,
producer_task=producer_info.task_type.
__name__,
goal=goal.name,
ordering=ordering))
else:
# We don't express dependencies on downstream tasks in this same goal.
pass
else:
goal_dependencies.add(producer_goal)
except round_manager.MissingProductError as e:
raise self.MissingProductError(
"Could not satisfy data dependencies for goal '{name}' with action {action}: {error}"
.format(name=task_name, action=task_type.__name__,
error=e))
goal_info = self.GoalInfo(goal, tasktypes_by_name, goal_dependencies)
goal_info_by_goal[goal] = goal_info
for goal_dependency in goal_dependencies:
self._visit_goal(goal_dependency, context, goal_info_by_goal)
def go_env(self, gopath=None): """Return an env dict that represents a proper Go environment mapping for this distribution.""" # Forcibly nullify the GOPATH if the command does not need one - this can prevent bad user # GOPATHs from erroring out commands; see: https://github.com/pantsbuild/pants/issues/2321. # NB: As of go 1.8, when GOPATH is unset (set to ''), it defaults to ~/go (assuming HOME is # set - and we can't unset that since it might legitimately be used by the subcommand); so we # set the GOPATH here to a valid value that nonetheless will fail to work if GOPATH is # actually used by the subcommand. no_gopath = os.devnull return OrderedDict(GOROOT=self.goroot, GOPATH=gopath or no_gopath)
def _topological_sort(self, goal_info_by_goal):
dependees_by_goal = OrderedDict()
def add_dependee(goal, dependee=None):
dependees = dependees_by_goal.get(goal)
if dependees is None:
dependees = set()
dependees_by_goal[goal] = dependees
if dependee:
dependees.add(dependee)
for goal, goal_info in goal_info_by_goal.items():
add_dependee(goal)
for dependency in goal_info.goal_dependencies:
add_dependee(dependency, goal)
satisfied = set()
while dependees_by_goal:
count = len(dependees_by_goal)
for goal, dependees in dependees_by_goal.items():
unsatisfied = len(dependees - satisfied)
if unsatisfied == 0:
satisfied.add(goal)
dependees_by_goal.pop(goal)
yield goal_info_by_goal[goal]
break
if len(dependees_by_goal) == count:
for dependees in dependees_by_goal.values():
dependees.difference_update(satisfied)
# TODO(John Sirois): Do a better job here and actually collect and print cycle paths
# between Goals/Tasks. The developer can most directly address that data.
raise self.GoalCycleError('Cycle detected in goal dependencies:\n\t{0}'
.format('\n\t'.join('{0} <- {1}'.format(goal, list(dependees))
for goal, dependees
in dependees_by_goal.items())))
def execute(self):
codegen_targets = self.codegen_targets()
if not codegen_targets:
return
self._validate_sources_globs()
with self.invalidated(codegen_targets,
invalidate_dependents=True,
topological_order=True,
fingerprint_strategy=self.get_fingerprint_strategy()) as invalidation_check:
with self.context.new_workunit(name='execute', labels=[WorkUnitLabel.MULTITOOL]):
vts_to_sources = OrderedDict()
for vt in invalidation_check.all_vts:
vts_to_sources[vt] = None
# Build the target and handle duplicate sources.
if not vt.valid:
if self._do_validate_sources_present(vt.target):
self.execute_codegen(vt.target, vt.current_results_dir)
sources = self._capture_sources((vt,))[0]
# _handle_duplicate_sources may delete files from the filesystem, so we need to
# re-capture the sources.
if not self._handle_duplicate_sources(vt, sources):
vts_to_sources[vt] = sources
vt.update()
vts_to_capture = tuple(key for key, sources in vts_to_sources.items() if sources is None)
filesets = self._capture_sources(vts_to_capture)
for key, fileset in zip(vts_to_capture, filesets):
vts_to_sources[key] = fileset
for vt, fileset in vts_to_sources.items():
self._inject_synthetic_target(vt, fileset)
self._mark_transitive_invalidation_hashes_dirty(
vt.target.address for vt in invalidation_check.all_vts
)
def _get_python_thrift_library_sources(self, py_thrift_targets):
"""Get file contents for python thrift library targets."""
target_snapshots = OrderedDict(
(t, t.sources_snapshot(
scheduler=self.context._scheduler).directory_digest)
for t in py_thrift_targets)
filescontent_by_target = OrderedDict(
zip(
target_snapshots.keys(),
self.context._scheduler.product_request(
FilesContent, target_snapshots.values())))
thrift_file_sources_by_target = OrderedDict(
(t, [(file_content.path, file_content.content)
for file_content in all_content.dependencies])
for t, all_content in filescontent_by_target.items())
return thrift_file_sources_by_target
def test_enum_resolve_variant(self):
one_enum_instance = SomeEnum(1)
two_enum_instance = SomeEnum(2)
self.assertEqual(
3, one_enum_instance.resolve_for_enum_variant({
1: 3,
2: 4,
}))
self.assertEqual(
4, two_enum_instance.resolve_for_enum_variant({
1: 3,
2: 4,
}))
# Test that an unrecognized variant raises an error.
with self.assertRaisesWithMessage(
EnumVariantSelectionError, """\
type check error in class SomeEnum: pattern matching must have exactly the keys [1, 2] (was: [1, 2, 3])"""
):
one_enum_instance.resolve_for_enum_variant({
1: 3,
2: 4,
3: 5,
})
# Test that not providing all the variants raises an error.
with self.assertRaisesWithMessage(
EnumVariantSelectionError, """\
type check error in class SomeEnum: pattern matching must have exactly the keys [1, 2] (was: [1])"""
):
one_enum_instance.resolve_for_enum_variant({
1: 3,
})
# Test that the ordering of the values in the enum constructor is not relevant for testing
# whether all variants are provided.
class OutOfOrderEnum(enum([2, 1, 3])):
pass
two_out_of_order_instance = OutOfOrderEnum(2)
# This OrderedDict mapping is in a different order than in the enum constructor. This test means
# we can rely on providing simply a literal dict to resolve_for_enum_variant() and not worry
# that the dict ordering will cause an error.
letter = two_out_of_order_instance.resolve_for_enum_variant(
OrderedDict([
(1, 'b'),
(2, 'a'),
(3, 'c'),
]))
self.assertEqual(letter, 'a')
def default(self, o):
if self._is_natively_encodable(o):
# isinstance() checks are expensive, particularly for abstract base classes such as Mapping:
# https://stackoverflow.com/questions/42378726/why-is-checking-isinstancesomething-mapping-so-slow
# This means that, if we let natively encodable types all through, we incur a performance hit, since
# we call this function very often.
# TODO(#7658) Figure out why we call this function so often.
return o
if isinstance(o, Mapping):
# Preserve order to avoid collisions for OrderedDict inputs to json.dumps(). We don't do this
# for general mappings because dicts have an arbitrary key ordering in some versions of python
# 3 (2.7 and 3.6-3.7 are known to have sorted keys, but with different definitions of sorted
# orders across versions, including insertion order). We want unordered dicts to collide if
# they have the same keys, in the same way we special-case sets below. Calling sorted() should
# be very fast if the keys happen to be pre-sorted. Pants options don't support OrderedDict
# inputs, and allowing them creates an ambiguity we don't need to deal with right now. See
# discussion in #6475.
if isinstance(o, OrderedDict):
raise TypeError('{cls} does not support OrderedDict inputs: {val!r}.'
.format(cls=type(self).__name__, val=o))
# TODO(#7082): we can remove the sorted() and OrderedDict when we drop python 2.7 and simply
# ensure we encode the keys/values as we do right here.
ordered_kv_pairs = sorted(o.items(), key=lambda x: x[0])
return OrderedDict(
(self._maybe_encode_dict_key(k), self.default(v))
for k, v in ordered_kv_pairs)
if isinstance(o, Set):
# We disallow OrderedSet (although it is not a stdlib collection) for the same reasons as
# OrderedDict above.
if isinstance(o, OrderedSet):
raise TypeError('{cls} does not support OrderedSet inputs: {val!r}.'
.format(cls=type(self).__name__, val=o))
# Set order is arbitrary in python 3.6 and 3.7, so we need to keep this sorted() call.
return sorted(self.default(i) for i in o)
if isinstance(o, DatatypeMixin):
# datatype objects will intentionally raise in the __iter__ method, but the Iterable abstract
# base class will match any class with any superclass that has the attribute __iter__ in the
# __dict__ (see https://docs.python.org/2/library/abc.html), so we need to check for it
# specially here.
# TODO: determine if the __repr__ should be some abstractmethod on DatatypeMixin!
return self.default(repr(o))
if isinstance(o, Iterable) and not isinstance(o, (bytes, list, str)):
return list(self.default(i) for i in o)
logger.debug("Our custom json encoder {} is trying to hash a primitive type, but has gone through"
"checking every other registered type class before. These checks are expensive,"
"so you should consider registering the type {} within"
"this function ({}.default)".format(type(self).__name__, type(o), type(self).__name__))
return o
def test_reduced_dependencies_2(self):
# foo --> baz
# | ^
# v |
# bar ----'
dep_map = OrderedDict(foo=['bar', 'baz'], bar=['baz'], baz=[])
target_map = self.create_dependencies(dep_map)
self.assertEqual(
self.dependency_calculator.reduced_dependencies(target_map['foo']),
OrderedSet([target_map['bar'], target_map['baz']]))
self.assertEqual(
self.dependency_calculator.reduced_dependencies(target_map['bar']),
OrderedSet([target_map['baz']]))
self.assertEqual(
self.dependency_calculator.reduced_dependencies(target_map['baz']),
OrderedSet())
def test_reduced_dependencies_1(self):
# foo -> bar -> baz
dep_map = OrderedDict(foo=['bar'], bar=['baz'], baz=[])
target_map = self.create_dependencies(dep_map)
self.assertEqual(
self.dependency_calculator.reduced_dependencies(target_map['foo']),
OrderedSet([target_map['bar']]))
self.assertEqual(
self.dependency_calculator.reduced_dependencies(target_map['bar']),
OrderedSet([target_map['baz']]))
self.assertEqual(
self.dependency_calculator.reduced_dependencies(target_map['baz']),
OrderedSet())
self.assert_requirements(target_map['foo'], {'bar==0.0.0'})
self.assert_requirements(target_map['bar'], {'baz==0.0.0'})
self.assert_requirements(target_map['baz'], set())
def _parse(lines):
def coalesce_lines():
line_iter = iter(lines)
try:
buffer = ''
while True:
line = next(line_iter)
if line.strip().endswith('\\'):
# Continuation.
buffer += line.strip()[:-1]
else:
if buffer:
# Continuation join, preserve left hand ws (could be a kv separator)
buffer += line.rstrip()
else:
# Plain old line
buffer = line.strip()
try:
yield buffer
finally:
buffer = ''
except StopIteration:
pass
def normalize(atom):
return re.sub(r'\\([:=\s])', r'\1', atom.strip())
def parse_line(line):
if line and not (line.startswith('#') or line.startswith('!')):
match = Properties._EXPLICIT_KV_SEP.search(line)
if match:
return normalize(line[:match.start()]), normalize(line[match.end():])
else:
space_sep = line.find(' ')
if space_sep == -1:
return normalize(line), ''
else:
return normalize(line[:space_sep]), normalize(line[space_sep:])
props = OrderedDict()
for line in coalesce_lines():
kv_pair = parse_line(line)
if kv_pair:
key, value = kv_pair
props[key] = value
return props
def _calculate_sources(self, target):
gentargets = OrderedSet()
def add_to_gentargets(tgt):
if self.is_gentarget(tgt):
gentargets.add(tgt)
self.context.build_graph.walk_transitive_dependency_graph(
[target.address], add_to_gentargets, postorder=True)
sources_by_base = OrderedDict()
for target in gentargets:
base = target.target_base
if base not in sources_by_base:
sources_by_base[base] = OrderedSet()
sources_by_base[base].update(
target.sources_relative_to_buildroot())
return sources_by_base
def _filemap(self, abs_path):
filemap = OrderedDict()
if self.fileset is not None:
paths = self.fileset() if isinstance(self.fileset, Fileset) \
else self.fileset if hasattr(self.fileset, '__iter__') \
else [self.fileset]
for path in paths:
if abs_path:
if not os.path.isabs(path):
path = os.path.join(get_buildroot(), self.rel_path,
path)
else:
if os.path.isabs(path):
path = fast_relpath(path, get_buildroot())
else:
path = os.path.join(self.rel_path, path)
filemap[path] = self.mapper(path)
return filemap
def _extract_all_python_namespaces(self, thrift_file_sources_by_target):
"""Extract the python namespace from each thrift source file."""
py_namespaces_by_target = OrderedDict()
failing_py_thrift_by_target = defaultdict(list)
for t, all_content in thrift_file_sources_by_target.items():
py_namespaces_by_target[t] = []
for (path, content) in all_content:
try:
py_namespaces_by_target[t].append(
# File content is provided as a binary string, so we have to decode it.
(path,
self._extract_py_namespace_from_content(
t, path, content.decode('utf-8'))))
except self.NamespaceParseFailure:
failing_py_thrift_by_target[t].append(path)
if failing_py_thrift_by_target:
# We dump the output to a file here because the output can be very long in some repos.
no_py_namespace_output_file = os.path.join(
self.workdir, 'no-python-namespace-output.txt')
pretty_printed_failures = '\n'.join(
'{}: [{}]'.format(t.address.spec, ', '.join(paths))
for t, paths in failing_py_thrift_by_target.items())
error = self.NamespaceExtractionError(
no_py_namespace_output_file, """\
Python namespaces could not be extracted from some thrift sources. Declaring a `namespace py` in
thrift sources for python thrift library targets will soon become required.
{} python library target(s) contained thrift sources not declaring a python namespace. The targets
and/or files which need to be edited will be dumped to: {}
""".format(len(failing_py_thrift_by_target), no_py_namespace_output_file))
safe_file_dump(no_py_namespace_output_file,
'{}\n'.format(pretty_printed_failures),
mode='w')
if self.get_options().strict:
raise error
else:
self.context.log.warn(error)
return py_namespaces_by_target
def test_reduced_dependencies_diamond(self):
# bar <-- foo --> baz
# | |
# `----> bak <----'
dep_map = OrderedDict(foo=['bar', 'baz'],
bar=['bak'],
baz=['bak'],
bak=[])
target_map = self.create_dependencies(dep_map)
self.assertEqual(
self.dependency_calculator.reduced_dependencies(target_map['foo']),
OrderedSet([target_map['bar'], target_map['baz']]))
self.assertEqual(
self.dependency_calculator.reduced_dependencies(target_map['bar']),
OrderedSet([target_map['bak']]))
self.assertEqual(
self.dependency_calculator.reduced_dependencies(target_map['baz']),
OrderedSet([target_map['bak']]))
self.assert_requirements(target_map['foo'],
{'bar==0.0.0', 'baz==0.0.0'})
self.assert_requirements(target_map['bar'], {'bak==0.0.0'})
self.assert_requirements(target_map['baz'], {'bak==0.0.0'})
def create(cls, spec_path, address_maps):
"""Creates an address family from the given set of address maps.
:param spec_path: The directory prefix shared by all address_maps.
:param address_maps: The family of maps that form this namespace.
:type address_maps: :class:`collections.Iterable` of :class:`AddressMap`
:returns: a new address family.
:rtype: :class:`AddressFamily`
:raises: :class:`MappingError` if the given address maps do not form a family.
"""
if spec_path == '.':
spec_path = ''
for address_map in address_maps:
if not address_map.path.startswith(spec_path):
raise DifferingFamiliesError(
'Expected AddressMaps to share the same parent directory {}, '
'but received: {}'.format(spec_path, address_map.path))
objects_by_name = {}
for address_map in address_maps:
current_path = address_map.path
for name, obj in address_map.objects_by_name.items():
previous = objects_by_name.get(name)
if previous:
previous_path, _ = previous
raise DuplicateNameError(
'An object with name {name!r} is already defined in '
'{previous_path!r}, will not overwrite with {obj!r} from '
'{current_path!r}.'.format(name=name,
previous_path=previous_path,
obj=obj,
current_path=current_path))
objects_by_name[name] = (current_path, obj)
return AddressFamily(
namespace=spec_path,
objects_by_name=OrderedDict(
(name, (path, obj))
for name, (path, obj) in sorted(objects_by_name.items())))
def create(cls, rule_entries):
"""Creates a RuleIndex with tasks indexed by their output type."""
serializable_rules = OrderedDict()
serializable_roots = OrderedSet()
def add_task(product_type, rule):
if product_type not in serializable_rules:
serializable_rules[product_type] = OrderedSet()
serializable_rules[product_type].add(rule)
def add_rule(rule):
if isinstance(rule, RootRule):
serializable_roots.add(rule)
return
# TODO: Ensure that interior types work by indexing on the list of types in
# the constraint. This heterogenity has some confusing implications:
# see https://github.com/pantsbuild/pants/issues/4005
for kind in rule.output_constraint.types:
add_task(kind, rule)
add_task(rule.output_constraint, rule)
for entry in rule_entries:
if isinstance(entry, Rule):
add_rule(entry)
elif hasattr(entry, '__call__'):
rule = getattr(entry, 'rule', None)
if rule is None:
raise TypeError(
"Expected callable {} to be decorated with @rule.".
format(entry))
add_rule(rule)
else:
raise TypeError(
"Unexpected rule type: {}. "
"Rules either extend Rule, or are static functions "
"decorated with @rule.".format(type(entry)))
return cls(serializable_rules, serializable_roots)
def get_arg_descriptions_from_docstring(cls, obj):
"""Returns an ordered map of arg name -> arg description found in :param: stanzas."""
ret = OrderedDict()
name = ''
doc = obj.__doc__ or ''
lines = [s.strip() for s in doc.split('\n')]
stanza_first_line_re = cls._get_stanza_first_line_re()
for line in lines:
m = stanza_first_line_re.match(line)
if m and m.group(1) == 'param':
# If first line of a parameter description, set name and description.
name, description = m.group(3, 4)
ret[name] = description
elif m and m.group(1) != 'param':
# If first line of a description of an item other than a parameter, clear name.
name = ''
elif name and line:
# If subsequent line of a parameter description, add to existing description (if any) for
# that parameter.
ret[name] += (' ' + line) if ret[name] else line
# Ignore subsequent lines of descriptions of items other than parameters.
return ret
def parse(cls, filepath, filecontent, parser):
"""Parses a source for addressable Serializable objects.
No matter the parser used, the parsed and mapped addressable objects are all 'thin'; ie: any
objects they point to in other namespaces or even in the same namespace but from a seperate
source are left as unresolved pointers.
:param string filepath: The path to the byte source containing serialized objects.
:param string filecontent: The content of byte source containing serialized objects to be parsed.
:param symbol_table: The symbol table cls to expose a symbol table dict.
:type symbol_table: Instance of :class:`pants.engine.parser.SymbolTable`.
:param parser: The parser cls to use.
:type parser: A :class:`pants.engine.parser.Parser`.
"""
try:
objects = parser.parse(filepath, filecontent)
except Exception as e:
raise MappingError('Failed to parse {}:\n{}'.format(filepath, e))
objects_by_name = {}
for obj in objects:
if not Serializable.is_serializable(obj):
raise UnaddressableObjectError(
'Parsed a non-serializable object: {!r}'.format(obj))
attributes = obj._asdict()
name = attributes.get('name')
if not name:
raise UnaddressableObjectError(
'Parsed a non-addressable object: {!r}'.format(obj))
if name in objects_by_name:
raise DuplicateNameError(
'An object already exists at {!r} with name {!r}: {!r}. Cannot '
'map {!r}'.format(filepath, name, objects_by_name[name],
obj))
objects_by_name[name] = obj
return cls(filepath, OrderedDict(sorted(objects_by_name.items())))
def _topological_sort(self, goal_info_by_goal):
dependees_by_goal = OrderedDict()
def add_dependee(goal, dependee=None):
dependees = dependees_by_goal.get(goal)
if dependees is None:
dependees = set()
dependees_by_goal[goal] = dependees
if dependee:
dependees.add(dependee)
for goal, goal_info in goal_info_by_goal.items():
add_dependee(goal)
for dependency in goal_info.goal_dependencies:
add_dependee(dependency, goal)
satisfied = set()
while dependees_by_goal:
count = len(dependees_by_goal)
for goal, dependees in dependees_by_goal.items():
unsatisfied = len(dependees - satisfied)
if unsatisfied == 0:
satisfied.add(goal)
dependees_by_goal.pop(goal)
yield goal_info_by_goal[goal]
break
if len(dependees_by_goal) == count:
for dependees in dependees_by_goal.values():
dependees.difference_update(satisfied)
# TODO(John Sirois): Do a better job here and actually collect and print cycle paths
# between Goals/Tasks. The developer can most directly address that data.
raise self.GoalCycleError(
'Cycle detected in goal dependencies:\n\t{0}'.format(
'\n\t'.join(
'{0} <- {1}'.format(goal, list(dependees))
for goal, dependees in dependees_by_goal.items())))
def _run_pytest(self, fail_fast, test_targets, workdirs):
if not test_targets:
return PytestResult.rc(0)
# Absolute path to chrooted test file -> Path to original test file relative to the buildroot.
sources_map = OrderedDict()
for t in test_targets:
for p in t.sources_relative_to_source_root():
sources_map[os.path.join(self._source_chroot_path,
p)] = os.path.join(t.target_base, p)
if not sources_map:
return PytestResult.rc(0)
with self._test_runner(workdirs, test_targets,
sources_map) as (pytest_binary, test_args,
get_pytest_rootdir):
# Validate that the user didn't provide any passthru args that conflict
# with those we must set ourselves.
for arg in self.get_passthru_args():
if arg.startswith('--junitxml') or arg.startswith(
'--confcutdir'):
raise TaskError(
'Cannot pass this arg through to pytest: {}'.format(
arg))
junitxml_path = workdirs.junitxml_path(*test_targets)
# N.B. the `--confcutdir` here instructs pytest to stop scanning for conftest.py files at the
# top of the buildroot. This prevents conftest.py files from outside (e.g. in users home dirs)
# from leaking into pants test runs. See: https://github.com/pantsbuild/pants/issues/2726
args = [
'-c', pytest_binary.config_path, '--junitxml', junitxml_path,
'--confcutdir',
get_buildroot(), '--continue-on-collection-errors'
]
if fail_fast:
args.extend(['-x'])
if self._debug:
args.extend(['-s'])
if self.get_options().colors:
args.extend(['--color', 'yes'])
if self.get_options().options:
for opt in self.get_options().options:
args.extend(safe_shlex_split(opt))
args.extend(self.get_passthru_args())
args.extend(test_args)
args.extend(sources_map.keys())
# We want to ensure our reporting based off junit xml is from this run so kill results from
# prior runs.
if os.path.exists(junitxml_path):
os.unlink(junitxml_path)
with self._maybe_run_in_chroot():
result = self._do_run_tests_with_args(pytest_binary.pex, args)
# There was a problem prior to test execution preventing junit xml file creation so just let
# the failure result bubble.
if not os.path.exists(junitxml_path):
return result
pytest_rootdir = get_pytest_rootdir()
failed_targets = self._get_failed_targets_from_junitxml(
junitxml_path, test_targets, pytest_rootdir)
def parse_error_handler(parse_error):
# Simple error handler to pass to xml parsing function.
raise TaskError('Error parsing xml file at {}: {}'.format(
parse_error.xml_path, parse_error.cause))
all_tests_info = self.parse_test_info(
junitxml_path, parse_error_handler,
['file', 'name', 'classname'])
for test_name, test_info in all_tests_info.items():
test_target = self._get_target_from_test(
test_info, test_targets, pytest_rootdir)
self.report_all_info_for_single_test(self.options_scope,
test_target, test_name,
test_info)
return result.with_failed_targets(failed_targets)
class RunInfo(object):
"""A little plaintext file containing very basic info about a pants run.
Can only be appended to, never edited.
"""
def __init__(self, info_file):
self._info_file = info_file
safe_mkdir_for(self._info_file)
self._info = OrderedDict()
if os.path.exists(self._info_file):
with open(self._info_file, 'r') as infile:
info = infile.read()
for m in re.finditer("""^([^:]+):(.*)$""", info, re.MULTILINE):
self._info[m.group(1).strip()] = m.group(2).strip()
def path(self):
return self._info_file
def get_info(self, key):
return self._info.get(key, None)
def __getitem__(self, key):
ret = self.get_info(key)
if ret is None:
raise KeyError(key)
return ret
def get_as_dict(self):
return self._info.copy()
def add_info(self, key, val, ignore_errors=False):
"""Adds the given info and returns a dict composed of just this added info."""
self.add_infos((key, val), ignore_errors=ignore_errors)
def add_infos(self, *keyvals, **kwargs):
"""Adds the given info and returns a dict composed of just this added info."""
kv_pairs = []
for key, val in keyvals:
key = key.strip()
val = str(val).strip()
if ':' in key:
raise ValueError(
'info key "{}" must not contain a colon.'.format(key))
kv_pairs.append((key, val))
for k, v in kv_pairs:
if k in self._info:
raise ValueError('info key "{}" already exists with value {}. '
'Cannot add it again with value {}.'.format(
k, self._info[k], v))
self._info[k] = v
try:
with open(self._info_file, 'a') as outfile:
for k, v in kv_pairs:
outfile.write('{}: {}\n'.format(k, v))
except IOError:
if not kwargs.get('ignore_errors', False):
raise
def add_basic_info(self, run_id, timestamp):
"""Adds basic build info."""
datetime = time.strftime('%A %b %d, %Y %H:%M:%S',
time.localtime(timestamp))
user = getpass.getuser()
machine = socket.gethostname()
buildroot = get_buildroot()
# TODO: Get rid of the redundant 'path' key once everyone is off it.
self.add_infos(('id', run_id), ('timestamp', timestamp),
('datetime', datetime), ('user', user),
('machine', machine), ('path', buildroot),
('buildroot', buildroot), ('version', version.VERSION))
def add_scm_info(self):
"""Adds SCM-related info."""
scm = get_scm()
if scm:
revision = scm.commit_id
branch = scm.branch_name or revision
else:
revision, branch = 'none', 'none'
self.add_infos(('revision', revision), ('branch', branch))
def _run_pytest(self, fail_fast, test_targets, workdirs):
if not test_targets:
return PytestResult.rc(0)
# Absolute path to chrooted test file -> Path to original test file relative to the buildroot.
sources_map = OrderedDict()
for t in test_targets:
for p in t.sources_relative_to_source_root():
sources_map[os.path.join(self._source_chroot_path, p)] = os.path.join(t.target_base, p)
if not sources_map:
return PytestResult.rc(0)
with self._test_runner(workdirs, test_targets, sources_map) as (pytest_binary,
test_args,
get_pytest_rootdir):
# Validate that the user didn't provide any passthru args that conflict
# with those we must set ourselves.
for arg in self.get_passthru_args():
if arg.startswith('--junitxml') or arg.startswith('--confcutdir'):
raise TaskError('Cannot pass this arg through to pytest: {}'.format(arg))
junitxml_path = workdirs.junitxml_path(*test_targets)
# N.B. the `--confcutdir` here instructs pytest to stop scanning for conftest.py files at the
# top of the buildroot. This prevents conftest.py files from outside (e.g. in users home dirs)
# from leaking into pants test runs. See: https://github.com/pantsbuild/pants/issues/2726
args = ['-c', pytest_binary.config_path,
'--junitxml', junitxml_path,
'--confcutdir', get_buildroot(),
'--continue-on-collection-errors']
if fail_fast:
args.extend(['-x'])
if self._debug:
args.extend(['-s'])
if self.get_options().colors:
args.extend(['--color', 'yes'])
if self.get_options().options:
for opt in self.get_options().options:
args.extend(safe_shlex_split(opt))
args.extend(self.get_passthru_args())
args.extend(test_args)
args.extend(sources_map.keys())
# We want to ensure our reporting based off junit xml is from this run so kill results from
# prior runs.
if os.path.exists(junitxml_path):
os.unlink(junitxml_path)
with self._maybe_run_in_chroot():
result = self._do_run_tests_with_args(pytest_binary.pex, args)
# There was a problem prior to test execution preventing junit xml file creation so just let
# the failure result bubble.
if not os.path.exists(junitxml_path):
return result
pytest_rootdir = get_pytest_rootdir()
failed_targets = self._get_failed_targets_from_junitxml(junitxml_path,
test_targets,
pytest_rootdir)
def parse_error_handler(parse_error):
# Simple error handler to pass to xml parsing function.
raise TaskError('Error parsing xml file at {}: {}'
.format(parse_error.xml_path, parse_error.cause))
all_tests_info = self.parse_test_info(junitxml_path, parse_error_handler,
['file', 'name', 'classname'])
for test_name, test_info in all_tests_info.items():
test_target = self._get_target_from_test(test_info, test_targets, pytest_rootdir)
self.report_all_info_for_single_test(self.options_scope, test_target, test_name, test_info)
return result.with_failed_targets(failed_targets)
class BuildGraph(AbstractClass):
"""A directed acyclic graph of Targets and dependencies. Not necessarily connected.
:API: public
"""
class DuplicateAddressError(AddressLookupError):
"""The same address appears multiple times in a dependency list
:API: public
"""
class TransitiveLookupError(AddressLookupError):
"""Used to append the current node to the error message from an AddressLookupError
:API: public
"""
class ManualSyntheticTargetError(AddressLookupError):
"""Used to indicate that an synthetic target was defined manually
:API: public
"""
def __init__(self, addr):
super(BuildGraph.ManualSyntheticTargetError, self).__init__(
'Found a manually-defined target at synthetic address {}'.format(addr.spec))
class NoDepPredicateWalk(object):
"""This is a utility class to aid in graph traversals that don't have predicates on dependency edges."""
def __init__(self):
self._worked = set()
self._expanded = set()
def expanded_or_worked(self, vertex):
"""Returns True if the vertex has been expanded or worked."""
return vertex in self._expanded or vertex in self._worked
def do_work_once(self, vertex):
"""Returns True exactly once for the given vertex."""
if vertex in self._worked:
return False
self._worked.add(vertex)
return True
def expand_once(self, vertex, _):
"""Returns True exactly once for the given vertex."""
if vertex in self._expanded:
return False
self._expanded.add(vertex)
return True
def dep_predicate(self, target, dep, level):
return True
class DepPredicateWalk(NoDepPredicateWalk):
"""This is a utility class to aid in graph traversals that don't care about the depth."""
def __init__(self, dep_predicate):
super(BuildGraph.DepPredicateWalk, self).__init__()
self._dep_predicate = dep_predicate
def dep_predicate(self, target, dep, level):
return self._dep_predicate(target, dep)
@staticmethod
def closure(*vargs, **kwargs):
"""See `Target.closure_for_targets` for arguments.
:API: public
"""
return Target.closure_for_targets(*vargs, **kwargs)
def __init__(self):
self.reset()
def __len__(self):
return len(self._target_by_address)
def target_file_count(self):
"""Returns a count of source files owned by all Targets in the BuildGraph."""
# TODO: Move this file counting into the `ProductGraph`.
return sum(t.sources_count() for t in self.targets())
@abstractmethod
def clone_new(self):
"""Returns a new BuildGraph instance of the same type and with the same __init__ params."""
def apply_injectables(self, targets):
"""Given an iterable of `Target` instances, apply their transitive injectables."""
target_types = {type(t) for t in targets}
target_subsystem_deps = {s for s in itertools.chain(*(t.subsystems() for t in target_types))}
for subsystem in target_subsystem_deps:
# TODO: The is_initialized() check is primarily for tests and would be nice to do away with.
if issubclass(subsystem, InjectablesMixin) and subsystem.is_initialized():
subsystem.global_instance().injectables(self)
def reset(self):
"""Clear out the state of the BuildGraph, in particular Target mappings and dependencies.
:API: public
"""
self._target_by_address = OrderedDict()
self._target_dependencies_by_address = defaultdict(OrderedSet)
self._target_dependees_by_address = defaultdict(OrderedSet)
self._derived_from_by_derivative = {} # Address -> Address.
self._derivatives_by_derived_from = defaultdict(list) # Address -> list of Address.
self.synthetic_addresses = set()
def contains_address(self, address):
"""
:API: public
"""
return address in self._target_by_address
def get_target_from_spec(self, spec, relative_to=''):
"""Converts `spec` into an address and returns the result of `get_target`
:API: public
"""
return self.get_target(Address.parse(spec, relative_to=relative_to))
def get_target(self, address):
"""Returns the Target at `address` if it has been injected into the BuildGraph, otherwise None.
:API: public
"""
return self._target_by_address.get(address, None)
def dependencies_of(self, address):
"""Returns the dependencies of the Target at `address`.
This method asserts that the address given is actually in the BuildGraph.
:API: public
"""
assert address in self._target_by_address, (
'Cannot retrieve dependencies of {address} because it is not in the BuildGraph.'
.format(address=address)
)
return self._target_dependencies_by_address[address]
def dependents_of(self, address):
"""Returns the addresses of the targets that depend on the target at `address`.
This method asserts that the address given is actually in the BuildGraph.
:API: public
"""
assert address in self._target_by_address, (
'Cannot retrieve dependents of {address} because it is not in the BuildGraph.'
.format(address=address)
)
return self._target_dependees_by_address[address]
def get_derived_from(self, address):
"""Get the target the specified target was derived from.
If a Target was injected programmatically, e.g. from codegen, this allows us to trace its
ancestry. If a Target is not derived, default to returning itself.
:API: public
"""
parent_address = self._derived_from_by_derivative.get(address, address)
return self.get_target(parent_address)
def get_concrete_derived_from(self, address):
"""Get the concrete target the specified target was (directly or indirectly) derived from.
The returned target is guaranteed to not have been derived from any other target.
:API: public
"""
current_address = address
next_address = self._derived_from_by_derivative.get(current_address, current_address)
while next_address != current_address:
current_address = next_address
next_address = self._derived_from_by_derivative.get(current_address, current_address)
return self.get_target(current_address)
def get_direct_derivatives(self, address):
"""Get all targets derived directly from the specified target.
Note that the specified target itself is not returned.
:API: public
"""
derivative_addrs = self._derivatives_by_derived_from.get(address, [])
return [self.get_target(addr) for addr in derivative_addrs]
def get_all_derivatives(self, address):
"""Get all targets derived directly or indirectly from the specified target.
Note that the specified target itself is not returned.
:API: public
"""
ret = []
direct = self.get_direct_derivatives(address)
ret.extend(direct)
for t in direct:
ret.extend(self.get_all_derivatives(t.address))
return ret
def inject_target(self, target, dependencies=None, derived_from=None, synthetic=False):
"""Injects a fully realized Target into the BuildGraph.
:API: public
:param Target target: The Target to inject.
:param list<Address> dependencies: The Target addresses that `target` depends on.
:param Target derived_from: The Target that `target` was derived from, usually as a result
of codegen.
:param bool synthetic: Whether to flag this target as synthetic, even if it isn't derived
from another target.
"""
if self.contains_address(target.address):
raise ValueError('Attempted to inject synthetic {target} derived from {derived_from}'
' into the BuildGraph with address {address}, but there is already a Target'
' {existing_target} with that address'
.format(target=target,
derived_from=derived_from,
address=target.address,
existing_target=self.get_target(target.address)))
dependencies = dependencies or frozenset()
address = target.address
if address in self._target_by_address:
raise ValueError('A Target {existing_target} already exists in the BuildGraph at address'
' {address}. Failed to insert {target}.'
.format(existing_target=self._target_by_address[address],
address=address,
target=target))
if derived_from:
if not self.contains_address(derived_from.address):
raise ValueError('Attempted to inject synthetic {target} derived from {derived_from}'
' into the BuildGraph, but {derived_from} was not in the BuildGraph.'
' Synthetic Targets must be derived from no Target (None) or from a'
' Target already in the BuildGraph.'
.format(target=target,
derived_from=derived_from))
self._derived_from_by_derivative[target.address] = derived_from.address
self._derivatives_by_derived_from[derived_from.address].append(target.address)
if derived_from or synthetic:
self.synthetic_addresses.add(address)
self._target_by_address[address] = target
for dependency_address in dependencies:
self.inject_dependency(dependent=address, dependency=dependency_address)
def inject_dependency(self, dependent, dependency):
"""Injects a dependency from `dependent` onto `dependency`.
It is an error to inject a dependency if the dependent doesn't already exist, but the reverse
is not an error.
:API: public
:param Address dependent: The (already injected) address of a Target to which `dependency`
is being added.
:param Address dependency: The dependency to be injected.
"""
if dependent not in self._target_by_address:
raise ValueError('Cannot inject dependency from {dependent} on {dependency} because the'
' dependent is not in the BuildGraph.'
.format(dependent=dependent, dependency=dependency))
# TODO(pl): Unfortunately this is an unhelpful time to error due to a cycle. Instead, we warn
# and allow the cycle to appear. It is the caller's responsibility to call sort_targets on the
# entire graph to generate a friendlier CycleException that actually prints the cycle.
# Alternatively, we could call sort_targets after every inject_dependency/inject_target, but
# that could have nasty performance implications. Alternative 2 would be to have an internal
# data structure of the topologically sorted graph which would have acceptable amortized
# performance for inserting new nodes, and also cycle detection on each insert.
if dependency not in self._target_by_address:
logger.warning('Injecting dependency from {dependent} on {dependency}, but the dependency'
' is not in the BuildGraph. This probably indicates a dependency cycle, but'
' it is not an error until sort_targets is called on a subgraph containing'
' the cycle.'
.format(dependent=dependent, dependency=dependency))
if dependency in self.dependencies_of(dependent):
logger.debug('{dependent} already depends on {dependency}'
.format(dependent=dependent, dependency=dependency))
else:
self._target_dependencies_by_address[dependent].add(dependency)
self._target_dependees_by_address[dependency].add(dependent)
def targets(self, predicate=None):
"""Returns all the targets in the graph in no particular order.
:API: public
:param predicate: A target predicate that will be used to filter the targets returned.
"""
return list(filter(predicate, self._target_by_address.values()))
def sorted_targets(self):
"""
:API: public
:return: targets ordered from most dependent to least.
"""
return sort_targets(self.targets())
def _walk_factory(self, dep_predicate):
"""Construct the right context object for managing state during a transitive walk."""
walk = None
if dep_predicate:
walk = self.DepPredicateWalk(dep_predicate)
else:
walk = self.NoDepPredicateWalk()
return walk
def walk_transitive_dependency_graph(self,
addresses,
work,
predicate=None,
postorder=False,
dep_predicate=None,
prelude=None,
epilogue=None):
"""Given a work function, walks the transitive dependency closure of `addresses` using DFS.
:API: public
:param list<Address> addresses: The closure of `addresses` will be walked.
:param function work: The function that will be called on every target in the closure using
the specified traversal order.
:param bool postorder: When ``True``, the traversal order is postorder (children before
parents), else it is preorder (parents before children).
:param function predicate: If this parameter is not given, no Targets will be filtered
out of the closure. If it is given, any Target which fails the predicate will not be
walked, nor will its dependencies. Thus predicate effectively trims out any subgraph
that would only be reachable through Targets that fail the predicate.
:param function dep_predicate: Takes two parameters, the current target and the dependency of
the current target. If this parameter is not given, no dependencies will be filtered
when traversing the closure. If it is given, when the predicate fails, the edge to the dependency
will not be expanded.
:param function prelude: Function to run before any dependency expansion.
It takes the currently explored target as an argument.
If ``postorder`` is ``False``, it will run after the current node is visited.
It is not affected by ``dep_predicate``.
It is not run if ``predicate`` does not succeed.
:param function epilogue: Function to run after children nodes are visited.
It takes the currently explored target as an argument.
If ``postorder`` is ``True``, it runs before visiting the current node.
It is not affected by ``dep_predicate``.
It is not run if ``predicate`` is not passed.
"""
walk = self._walk_factory(dep_predicate)
def _walk_rec(addr, level=0):
# If we've followed an edge to this address, stop recursing.
if not walk.expand_once(addr, level):
return
target = self._target_by_address[addr]
if predicate and not predicate(target):
return
if not postorder and walk.do_work_once(addr):
work(target)
if prelude:
prelude(target)
for dep_address in self._target_dependencies_by_address[addr]:
if walk.expanded_or_worked(dep_address):
continue
if walk.dep_predicate(target, self._target_by_address[dep_address], level):
_walk_rec(dep_address, level + 1)
if epilogue:
epilogue(target)
if postorder and walk.do_work_once(addr):
work(target)
for address in addresses:
_walk_rec(address)
def walk_transitive_dependee_graph(self,
addresses,
work,
predicate=None,
postorder=False,
prelude=None,
epilogue=None):
"""Identical to `walk_transitive_dependency_graph`, but walks dependees preorder (or postorder
if the postorder parameter is True).
This is identical to reversing the direction of every arrow in the DAG, then calling
`walk_transitive_dependency_graph`.
:API: public
"""
walked = set()
def _walk_rec(addr):
if addr not in walked:
walked.add(addr)
target = self._target_by_address[addr]
if not predicate or predicate(target):
if not postorder:
work(target)
if prelude:
prelude(target)
for dep_address in self._target_dependees_by_address[addr]:
_walk_rec(dep_address)
if epilogue:
epilogue(target)
if postorder:
work(target)
for address in addresses:
_walk_rec(address)
def transitive_dependees_of_addresses(self, addresses, predicate=None, postorder=False):
"""Returns all transitive dependees of `addresses`.
Note that this uses `walk_transitive_dependee_graph` and the predicate is passed through,
hence it trims graphs rather than just filtering out Targets that do not match the predicate.
See `walk_transitive_dependee_graph for more detail on `predicate`.
:API: public
:param list<Address> addresses: The root addresses to transitively close over.
:param function predicate: The predicate passed through to `walk_transitive_dependee_graph`.
"""
ret = OrderedSet()
self.walk_transitive_dependee_graph(addresses, ret.add, predicate=predicate,
postorder=postorder)
return ret
def transitive_subgraph_of_addresses(self, addresses, *vargs, **kwargs):
"""Returns all transitive dependencies of `addresses`.
Note that this uses `walk_transitive_dependencies_graph` and the predicate is passed through,
hence it trims graphs rather than just filtering out Targets that do not match the predicate.
See `walk_transitive_dependency_graph for more detail on `predicate`.
:API: public
:param list<Address> addresses: The root addresses to transitively close over.
:param function predicate: The predicate passed through to
`walk_transitive_dependencies_graph`.
:param bool postorder: When ``True``, the traversal order is postorder (children before
parents), else it is preorder (parents before children).
:param function predicate: If this parameter is not given, no Targets will be filtered
out of the closure. If it is given, any Target which fails the predicate will not be
walked, nor will its dependencies. Thus predicate effectively trims out any subgraph
that would only be reachable through Targets that fail the predicate.
:param function dep_predicate: Takes two parameters, the current target and the dependency of
the current target. If this parameter is not given, no dependencies will be filtered
when traversing the closure. If it is given, when the predicate fails, the edge to the dependency
will not be expanded.
"""
ret = OrderedSet()
self.walk_transitive_dependency_graph(addresses, ret.add,
*vargs,
**kwargs)
return ret
def transitive_subgraph_of_addresses_bfs(self,
addresses,
predicate=None,
dep_predicate=None):
"""Returns the transitive dependency closure of `addresses` using BFS.
:API: public
:param list<Address> addresses: The closure of `addresses` will be walked.
:param function predicate: If this parameter is not given, no Targets will be filtered
out of the closure. If it is given, any Target which fails the predicate will not be
walked, nor will its dependencies. Thus predicate effectively trims out any subgraph
that would only be reachable through Targets that fail the predicate.
:param function dep_predicate: Takes two parameters, the current target and the dependency of
the current target. If this parameter is not given, no dependencies will be filtered
when traversing the closure. If it is given, when the predicate fails, the edge to the dependency
will not be expanded.
"""
walk = self._walk_factory(dep_predicate)
ordered_closure = OrderedSet()
to_walk = deque((0, addr) for addr in addresses)
while len(to_walk) > 0:
level, address = to_walk.popleft()
if not walk.expand_once(address, level):
continue
target = self._target_by_address[address]
if predicate and not predicate(target):
continue
if walk.do_work_once(address):
ordered_closure.add(target)
for dep_address in self._target_dependencies_by_address[address]:
if walk.expanded_or_worked(dep_address):
continue
if walk.dep_predicate(target, self._target_by_address[dep_address], level):
to_walk.append((level + 1, dep_address))
return ordered_closure
@abstractmethod
def inject_synthetic_target(self,
address,
target_type,
dependencies=None,
derived_from=None,
**kwargs):
"""Constructs and injects Target at `address` with optional `dependencies` and `derived_from`.
This method is useful especially for codegen, where a "derived" Target is injected
programmatically rather than read in from a BUILD file.
:API: public
:param Address address: The address of the new Target. Must not already be in the BuildGraph.
:param type target_type: The class of the Target to be constructed.
:param list<Address> dependencies: The dependencies of this Target, usually inferred or copied
from the `derived_from`.
:param Target derived_from: The Target this Target will derive from.
"""
def maybe_inject_address_closure(self, address):
"""If the given address is not already injected to the graph, calls inject_address_closure.
:API: public
:param Address address: The address to inject. Must be resolvable by `self._address_mapper` or
else be the address of an already injected entity.
"""
if not self.contains_address(address):
self.inject_address_closure(address)
@abstractmethod
def inject_address_closure(self, address):
"""Resolves, constructs and injects a Target and its transitive closure of dependencies.
This method is idempotent and will short circuit for already injected addresses. For all other
addresses though, it delegates to an internal AddressMapper to resolve item the address points
to.
:API: public
:param Address address: The address to inject. Must be resolvable by `self._address_mapper` or
else be the address of an already injected entity.
"""
@abstractmethod
def inject_specs_closure(self, specs, fail_fast=None):
"""Resolves, constructs and injects Targets and their transitive closures of dependencies.
:API: public
:param specs: A list of base.specs.Spec objects to resolve and inject.
:param fail_fast: Whether to fail quickly for the first error, or to complete all
possible injections before failing.
:returns: Yields a sequence of resolved Address objects.
"""
def resolve(self, spec):
"""Returns an iterator over the target(s) the given address points to."""
address = Address.parse(spec)
# NB: This is an idempotent, short-circuiting call.
self.inject_address_closure(address)
return self.transitive_subgraph_of_addresses([address])
@abstractmethod
def resolve_address(self, address):
"""Maps an address in the virtual address space to an object.
full_path = os.path.join(root, file)
relpath = os.path.relpath(full_path, basedir)
if prefix:
relpath = os.path.join(ensure_text(prefix), relpath)
zip.write(full_path, relpath)
return zippath
TAR = TarArchiver('w:', 'tar')
TGZ = TarArchiver('w:gz', 'tar.gz')
TBZ2 = TarArchiver('w:bz2', 'tar.bz2')
ZIP = ZipArchiver(ZIP_DEFLATED, 'zip')
_ARCHIVER_BY_TYPE = OrderedDict(
tar=TAR,
tgz=TGZ,
tbz2=TBZ2,
zip=ZIP)
archive_extensions = {
name: archiver.extension for name, archiver in _ARCHIVER_BY_TYPE.items()
}
TYPE_NAMES = frozenset(_ARCHIVER_BY_TYPE.keys())
TYPE_NAMES_NO_PRESERVE_SYMLINKS = frozenset({'zip'})
TYPE_NAMES_PRESERVE_SYMLINKS = TYPE_NAMES - TYPE_NAMES_NO_PRESERVE_SYMLINKS
def create_archiver(typename):
"""Returns Archivers in common configurations.
def sort_goals(self, context, goals):
goal_info_by_goal = OrderedDict()
for goal in reversed(OrderedSet(goals)):
self._visit_goal(goal, context, goal_info_by_goal)
return list(reversed(list(self._topological_sort(goal_info_by_goal))))
