def create(cls, rule_entries, union_rules=None):
"""Creates a RuleIndex with tasks indexed by their output type."""
serializable_rules = OrderedDict()
serializable_roots = OrderedSet()
union_rules = OrderedDict(union_rules or ())
def add_task(product_type, rule):
# TODO(#7311): make a defaultdict-like wrapper for OrderedDict if more widely used.
if product_type not in serializable_rules:
serializable_rules[product_type] = OrderedSet()
serializable_rules[product_type].add(rule)
def add_root_rule(root_rule):
serializable_roots.add(root_rule)
def add_rule(rule):
if isinstance(rule, RootRule):
add_root_rule(rule)
else:
add_task(rule.output_type, rule)
for dep_rule in rule.dependency_rules:
add_rule(dep_rule)
def add_type_transition_rule(union_rule):
# NB: This does not require that union bases be supplied to `def rules():`, as the union type
# is never instantiated!
union_base = union_rule.union_base
assert union_base._is_union
union_member = union_rule.union_member
if union_base not in union_rules:
union_rules[union_base] = OrderedSet()
union_rules[union_base].add(union_member)
for entry in rule_entries:
if isinstance(entry, Rule):
add_rule(entry)
elif isinstance(entry, UnionRule):
add_type_transition_rule(entry)
elif hasattr(entry, '__call__'):
rule = getattr(entry, 'rule', None)
if rule is None:
raise TypeError(
"Expected callable {} to be decorated with @rule.".
format(entry))
add_rule(rule)
else:
raise TypeError("""\
Rule entry {} had an unexpected type: {}. Rules either extend Rule or UnionRule, or are static \
functions decorated with @rule.""".format(entry, type(entry)))
return cls(serializable_rules, serializable_roots, union_rules)
def _singletons(cls):
"""Generate memoized instances of this enum wrapping each of this enum's allowed values.
NB: The implementation of enum() should use this property as the source of truth for allowed
values and enum instances from those values.
"""
return OrderedDict((value, cls._make_singleton(value)) for value in all_values_realized)
def classpath_by_targets(cls,
targets,
classpath_products,
confs=('default', )):
"""Return classpath entries grouped by their targets for the given `targets`.
:param targets: The targets to lookup classpath products for.
:param ClasspathProducts classpath_products: Product containing classpath elements.
:param confs: The list of confs for use by this classpath.
:returns: The ordered (target, classpath) mappings.
:rtype: OrderedDict
"""
classpath_target_tuples = classpath_products.get_product_target_mappings_for_targets(
targets)
filtered_items_iter = filter(
cls._accept_conf_filter(confs, lambda x: x[0][0]),
classpath_target_tuples)
# group (classpath_entry, target) tuples by targets
target_to_classpath = OrderedDict()
for classpath_entry, target in filtered_items_iter:
_, entry = classpath_entry
if not target in target_to_classpath:
target_to_classpath[target] = []
target_to_classpath[target].append(entry)
return target_to_classpath
def test_classpath_by_targets(self):
b = self.make_target('b', JvmTarget)
a = self.make_target('a',
JvmTarget,
dependencies=[b],
excludes=[Exclude('com.example', 'lib')])
classpath_products = ClasspathProducts(self.pants_workdir)
path1 = self._path('jar/path1')
path2 = self._path('jar/path2')
path3 = os.path.join(self.pants_workdir, 'jar/path3')
resolved_jar = ResolvedJar(M2Coordinate(org='com.example',
name='lib',
rev='1.0'),
cache_path='somewhere',
pants_path=path3)
classpath_products.add_for_target(a, [('default', path1)])
classpath_products.add_for_target(a, [('non-default', path2)])
classpath_products.add_for_target(b, [('default', path2)])
classpath_products.add_jars_for_targets([b], 'default', [resolved_jar])
classpath_products.add_excludes_for_targets([a])
# (a, path2) filtered because of conf
# (b, path3) filtered because of excludes
self.assertEqual(
OrderedDict([(a, [ClasspathEntry(path1)]),
(b, [ClasspathEntry(path2)])]),
ClasspathUtil.classpath_by_targets(a.closure(bfs=True),
classpath_products))
def default(self, o):
if isinstance(o, Mapping):
# Preserve order to avoid collisions for OrderedDict inputs to json.dumps(). We don't do this
# for general mappings because dicts have an arbitrary key ordering in some versions of python
# 3 (2.7 and 3.6-3.7 are known to have sorted keys, but with different definitions of sorted
# orders across versions, including insertion order). We want unordered dicts to collide if
# they have the same keys, in the same way we special-case sets below. Calling sorted() should
# be very fast if the keys happen to be pre-sorted. Pants options don't support OrderedDict
# inputs, and allowing them creates an ambiguity we don't need to deal with right now. See
# discussion in #6475.
if isinstance(o, OrderedDict):
raise TypeError(
'{cls} does not support OrderedDict inputs: {val!r}.'.
format(cls=type(self).__name__, val=o))
# TODO(#7082): we can remove the sorted() and OrderedDict when we drop python 2.7 and simply
# ensure we encode the keys/values as we do right here.
ordered_kv_pairs = sorted(o.items(), key=lambda x: x[0])
return OrderedDict(
(self._maybe_encode_dict_key(k), self.default(v))
for k, v in ordered_kv_pairs)
elif isinstance(o, Set):
# We disallow OrderedSet (although it is not a stdlib collection) for the same reasons as
# OrderedDict above.
if isinstance(o, OrderedSet):
raise TypeError(
'{cls} does not support OrderedSet inputs: {val!r}.'.
format(cls=type(self).__name__, val=o))
# Set order is arbitrary in python 3.6 and 3.7, so we need to keep this sorted() call.
return sorted(self.default(i) for i in o)
elif isinstance(o, Iterable) and not isinstance(o, (bytes, list, str)):
return list(self.default(i) for i in o)
return o
def _get_python_thrift_library_sources(self, py_thrift_targets):
"""Get file contents for python thrift library targets."""
target_snapshots = OrderedDict(
(t, t.sources_snapshot(
scheduler=self.context._scheduler).directory_digest)
for t in py_thrift_targets)
filescontent_by_target = OrderedDict(
zip(
target_snapshots.keys(),
self.context._scheduler.product_request(
FilesContent, target_snapshots.values())))
thrift_file_sources_by_target = OrderedDict(
(t, [(file_content.path, file_content.content)
for file_content in all_content.dependencies])
for t, all_content in filescontent_by_target.items())
return thrift_file_sources_by_target
def test_execution_reduced_dependencies_1(self):
dep_map = OrderedDict(foo=['bar'], bar=['baz'], baz=[])
target_map = self.create_dependencies(dep_map)
with self.run_execute(target_map['foo'], recursive=False) as created:
self.assertEqual([target_map['foo']], list(created.keys()))
with self.run_execute(target_map['foo'], recursive=True) as created:
self.assertEqual(
{target_map['baz'], target_map['bar'], target_map['foo']},
set(created.keys()))
def __init__(self, info_file):
self._info_file = info_file
safe_mkdir_for(self._info_file)
self._info = OrderedDict()
if os.path.exists(self._info_file):
with open(self._info_file, 'r') as infile:
info = infile.read()
for m in re.finditer("""^([^:]+):(.*)$""", info, re.MULTILINE):
self._info[m.group(1).strip()] = m.group(2).strip()
def _handle_duplicate_sources(self, vt, sources):
"""Handles duplicate sources generated by the given gen target by either failure or deletion.
This method should be called after all dependencies have been injected into the graph, but
before injecting the synthetic version of this target.
Returns a boolean indicating whether it modified the underlying filesystem.
NB(gm): Some code generators may re-generate code that their dependent libraries generate.
This results in targets claiming to generate sources that they really don't, so we try to
filter out sources that were actually generated by dependencies of the target. This causes
the code generated by the dependencies to 'win' over the code generated by dependees. By
default, this behavior is disabled, and duplication in generated sources will raise a
TaskError. This is controlled by the --allow-dups flag.
"""
target = vt.target
target_workdir = vt.results_dir
# Walk dependency gentargets and record any sources owned by those targets that are also
# owned by this target.
duplicates_by_target = OrderedDict()
def record_duplicates(dep):
if dep == target or not self.is_gentarget(dep.concrete_derived_from):
return False
duped_sources = [s for s in dep.sources_relative_to_source_root() if s in sources.files and
not self.ignore_dup(target, dep, s)]
if duped_sources:
duplicates_by_target[dep] = duped_sources
target.walk(record_duplicates)
# If there were no dupes, we're done.
if not duplicates_by_target:
return False
# If there were duplicates warn or error.
messages = ['{target} generated sources that had already been generated by dependencies.'
.format(target=target.address.spec)]
for dep, duped_sources in duplicates_by_target.items():
messages.append('\t{} also generated:'.format(dep.concrete_derived_from.address.spec))
messages.extend(['\t\t{}'.format(source) for source in duped_sources])
message = '\n'.join(messages)
if self.get_options().allow_dups:
logger.warn(message)
else:
raise self.DuplicateSourceError(message)
did_modify = False
# Finally, remove duplicates from the workdir. This prevents us from having to worry
# about them during future incremental compiles.
for dep, duped_sources in duplicates_by_target.items():
for duped_source in duped_sources:
safe_delete(os.path.join(target_workdir, duped_source))
did_modify = True
if did_modify:
Digest.clear(vt.current_results_dir)
return did_modify
def _visit_goal(self, goal, context, goal_info_by_goal):
if goal in goal_info_by_goal:
return
tasktypes_by_name = OrderedDict()
goal_dependencies = set()
visited_task_types = set()
for task_name in reversed(goal.ordered_task_names()):
task_type = goal.task_type_by_name(task_name)
tasktypes_by_name[task_name] = task_type
visited_task_types.add(task_type)
round_manager = RoundManager(context)
task_type.invoke_prepare(context.options, round_manager)
try:
dependencies = round_manager.get_dependencies()
for producer_info in dependencies:
producer_goal = producer_info.goal
if producer_goal == goal:
if producer_info.task_type == task_type:
# We allow a task to produce products it itself needs. We trust the Task writer
# to arrange for proper sequencing.
pass
elif producer_info.task_type in visited_task_types:
ordering = '\n\t'.join(
"[{0}] '{1}' {2}".format(
i, tn,
goal.task_type_by_name(tn).__name__)
for i, tn in enumerate(
goal.ordered_task_names()))
raise self.TaskOrderError(
"TaskRegistrar '{name}' with action {consumer_task} depends on {data} from task "
"{producer_task} which is ordered after it in the '{goal}' goal:\n\t{ordering}"
.format(name=task_name,
consumer_task=task_type.__name__,
data=producer_info.product_type,
producer_task=producer_info.task_type.
__name__,
goal=goal.name,
ordering=ordering))
else:
# We don't express dependencies on downstream tasks in this same goal.
pass
else:
goal_dependencies.add(producer_goal)
except round_manager.MissingProductError as e:
raise self.MissingProductError(
"Could not satisfy data dependencies for goal '{name}' with action {action}: {error}"
.format(name=task_name, action=task_type.__name__,
error=e))
goal_info = self.GoalInfo(goal, tasktypes_by_name, goal_dependencies)
goal_info_by_goal[goal] = goal_info
for goal_dependency in goal_dependencies:
self._visit_goal(goal_dependency, context, goal_info_by_goal)
def go_env(self, gopath=None): """Return an env dict that represents a proper Go environment mapping for this distribution.""" # Forcibly nullify the GOPATH if the command does not need one - this can prevent bad user # GOPATHs from erroring out commands; see: https://github.com/pantsbuild/pants/issues/2321. # NB: As of go 1.8, when GOPATH is unset (set to ''), it defaults to ~/go (assuming HOME is # set - and we can't unset that since it might legitimately be used by the subcommand); so we # set the GOPATH here to a valid value that nonetheless will fail to work if GOPATH is # actually used by the subcommand. no_gopath = os.devnull return OrderedDict(GOROOT=self.goroot, GOPATH=gopath or no_gopath)
def reset(self):
"""Clear out the state of the BuildGraph, in particular Target mappings and dependencies.
:API: public
"""
self._target_by_address = OrderedDict()
self._target_dependencies_by_address = defaultdict(OrderedSet)
self._target_dependees_by_address = defaultdict(OrderedSet)
self._derived_from_by_derivative = {} # Address -> Address.
self._derivatives_by_derived_from = defaultdict(list) # Address -> list of Address.
self.synthetic_addresses = set()
def execute(self):
codegen_targets = self.codegen_targets()
if not codegen_targets:
return
self._validate_sources_globs()
with self.invalidated(
codegen_targets,
invalidate_dependents=True,
topological_order=True,
fingerprint_strategy=self.get_fingerprint_strategy(
)) as invalidation_check:
with self.context.new_workunit(name='execute',
labels=[WorkUnitLabel.MULTITOOL]):
vts_to_sources = OrderedDict()
for vt in invalidation_check.all_vts:
synthetic_target_dir = self.synthetic_target_dir(
vt.target, vt.results_dir)
key = (vt, synthetic_target_dir)
vts_to_sources[key] = None
# Build the target and handle duplicate sources.
if not vt.valid:
if self._do_validate_sources_present(vt.target):
self.execute_codegen(vt.target, vt.results_dir)
sources = self._capture_sources((key, ))[0]
# _handle_duplicate_sources may delete files from the filesystem, so we need to
# re-capture the sources.
if not self._handle_duplicate_sources(
vt.target, vt.results_dir, sources):
vts_to_sources[key] = sources
vt.update()
vts_to_capture = tuple(
key for key, sources in vts_to_sources.items()
if sources is None)
filesets = self._capture_sources(vts_to_capture)
for key, fileset in zip(vts_to_capture, filesets):
vts_to_sources[key] = fileset
for (vt,
synthetic_target_dir), fileset in vts_to_sources.items():
self._inject_synthetic_target(
vt.target,
synthetic_target_dir,
fileset,
)
self._mark_transitive_invalidation_hashes_dirty(
vt.target.address for vt in invalidation_check.all_vts)
def test_enum_resolve_variant(self):
one_enum_instance = SomeEnum(1)
two_enum_instance = SomeEnum(2)
self.assertEqual(
3, one_enum_instance.resolve_for_enum_variant({
1: 3,
2: 4,
}))
self.assertEqual(
4, two_enum_instance.resolve_for_enum_variant({
1: 3,
2: 4,
}))
# Test that an unrecognized variant raises an error.
with self.assertRaisesWithMessage(
EnumVariantSelectionError, """\
type check error in class SomeEnum: pattern matching must have exactly the keys [1, 2] (was: [1, 2, 3])"""
):
one_enum_instance.resolve_for_enum_variant({
1: 3,
2: 4,
3: 5,
})
# Test that not providing all the variants raises an error.
with self.assertRaisesWithMessage(
EnumVariantSelectionError, """\
type check error in class SomeEnum: pattern matching must have exactly the keys [1, 2] (was: [1])"""
):
one_enum_instance.resolve_for_enum_variant({
1: 3,
})
# Test that the ordering of the values in the enum constructor is not relevant for testing
# whether all variants are provided.
class OutOfOrderEnum(enum([2, 1, 3])):
pass
two_out_of_order_instance = OutOfOrderEnum(2)
# This OrderedDict mapping is in a different order than in the enum constructor. This test means
# we can rely on providing simply a literal dict to resolve_for_enum_variant() and not worry
# that the dict ordering will cause an error.
letter = two_out_of_order_instance.resolve_for_enum_variant(
OrderedDict([
(1, 'b'),
(2, 'a'),
(3, 'c'),
]))
self.assertEqual(letter, 'a')
def default(self, o):
if self._is_natively_encodable(o):
# isinstance() checks are expensive, particularly for abstract base classes such as Mapping:
# https://stackoverflow.com/questions/42378726/why-is-checking-isinstancesomething-mapping-so-slow
# This means that, if we let natively encodable types all through, we incur a performance hit, since
# we call this function very often.
# TODO(#7658) Figure out why we call this function so often.
return o
if isinstance(o, Mapping):
# Preserve order to avoid collisions for OrderedDict inputs to json.dumps(). We don't do this
# for general mappings because dicts have an arbitrary key ordering in some versions of python
# 3 (2.7 and 3.6-3.7 are known to have sorted keys, but with different definitions of sorted
# orders across versions, including insertion order). We want unordered dicts to collide if
# they have the same keys, in the same way we special-case sets below. Calling sorted() should
# be very fast if the keys happen to be pre-sorted. Pants options don't support OrderedDict
# inputs, and allowing them creates an ambiguity we don't need to deal with right now. See
# discussion in #6475.
if isinstance(o, OrderedDict):
raise TypeError('{cls} does not support OrderedDict inputs: {val!r}.'
.format(cls=type(self).__name__, val=o))
# TODO(#7082): we can remove the sorted() and OrderedDict when we drop python 2.7 and simply
# ensure we encode the keys/values as we do right here.
ordered_kv_pairs = sorted(o.items(), key=lambda x: x[0])
return OrderedDict(
(self._maybe_encode_dict_key(k), self.default(v))
for k, v in ordered_kv_pairs)
if isinstance(o, Set):
# We disallow OrderedSet (although it is not a stdlib collection) for the same reasons as
# OrderedDict above.
if isinstance(o, OrderedSet):
raise TypeError('{cls} does not support OrderedSet inputs: {val!r}.'
.format(cls=type(self).__name__, val=o))
# Set order is arbitrary in python 3.6 and 3.7, so we need to keep this sorted() call.
return sorted(self.default(i) for i in o)
if isinstance(o, DatatypeMixin):
# datatype objects will intentionally raise in the __iter__ method, but the Iterable abstract
# base class will match any class with any superclass that has the attribute __iter__ in the
# __dict__ (see https://docs.python.org/2/library/abc.html), so we need to check for it
# specially here.
# TODO: determine if the __repr__ should be some abstractmethod on DatatypeMixin!
return self.default(repr(o))
if isinstance(o, Iterable) and not isinstance(o, (bytes, list, str)):
return list(self.default(i) for i in o)
logger.debug("Our custom json encoder {} is trying to hash a primitive type, but has gone through"
"checking every other registered type class before. These checks are expensive,"
"so you should consider registering the type {} within"
"this function ({}.default)".format(type(self).__name__, type(o), type(self).__name__))
return o
def test_reduced_dependencies_2(self):
# foo --> baz
# | ^
# v |
# bar ----'
dep_map = OrderedDict(foo=['bar', 'baz'], bar=['baz'], baz=[])
target_map = self.create_dependencies(dep_map)
self.assertEqual(
self.dependency_calculator.reduced_dependencies(target_map['foo']),
OrderedSet([target_map['bar'], target_map['baz']]))
self.assertEqual(
self.dependency_calculator.reduced_dependencies(target_map['bar']),
OrderedSet([target_map['baz']]))
self.assertEqual(
self.dependency_calculator.reduced_dependencies(target_map['baz']),
OrderedSet())
def test_reduced_dependencies_1(self):
# foo -> bar -> baz
dep_map = OrderedDict(foo=['bar'], bar=['baz'], baz=[])
target_map = self.create_dependencies(dep_map)
self.assertEqual(
self.dependency_calculator.reduced_dependencies(target_map['foo']),
OrderedSet([target_map['bar']]))
self.assertEqual(
self.dependency_calculator.reduced_dependencies(target_map['bar']),
OrderedSet([target_map['baz']]))
self.assertEqual(
self.dependency_calculator.reduced_dependencies(target_map['baz']),
OrderedSet())
self.assert_requirements(target_map['foo'], {'bar==0.0.0'})
self.assert_requirements(target_map['bar'], {'baz==0.0.0'})
self.assert_requirements(target_map['baz'], set())
def _parse(lines):
def coalesce_lines():
line_iter = iter(lines)
try:
buffer = ''
while True:
line = next(line_iter)
if line.strip().endswith('\\'):
# Continuation.
buffer += line.strip()[:-1]
else:
if buffer:
# Continuation join, preserve left hand ws (could be a kv separator)
buffer += line.rstrip()
else:
# Plain old line
buffer = line.strip()
try:
yield buffer
finally:
buffer = ''
except StopIteration:
pass
def normalize(atom):
return re.sub(r'\\([:=\s])', r'\1', atom.strip())
def parse_line(line):
if line and not (line.startswith('#') or line.startswith('!')):
match = Properties._EXPLICIT_KV_SEP.search(line)
if match:
return normalize(line[:match.start()]), normalize(line[match.end():])
else:
space_sep = line.find(' ')
if space_sep == -1:
return normalize(line), ''
else:
return normalize(line[:space_sep]), normalize(line[space_sep:])
props = OrderedDict()
for line in coalesce_lines():
kv_pair = parse_line(line)
if kv_pair:
key, value = kv_pair
props[key] = value
return props
def _calculate_sources(self, target):
gentargets = OrderedSet()
def add_to_gentargets(tgt):
if self.is_gentarget(tgt):
gentargets.add(tgt)
self.context.build_graph.walk_transitive_dependency_graph(
[target.address], add_to_gentargets, postorder=True)
sources_by_base = OrderedDict()
for target in gentargets:
base = target.target_base
if base not in sources_by_base:
sources_by_base[base] = OrderedSet()
sources_by_base[base].update(
target.sources_relative_to_buildroot())
return sources_by_base
def _filemap(self, abs_path):
filemap = OrderedDict()
if self.fileset is not None:
paths = self.fileset() if isinstance(self.fileset, Fileset) \
else self.fileset if hasattr(self.fileset, '__iter__') \
else [self.fileset]
for path in paths:
if abs_path:
if not os.path.isabs(path):
path = os.path.join(get_buildroot(), self.rel_path,
path)
else:
if os.path.isabs(path):
path = fast_relpath(path, get_buildroot())
else:
path = os.path.join(self.rel_path, path)
filemap[path] = self.mapper(path)
return filemap
def _extract_all_python_namespaces(self, thrift_file_sources_by_target):
"""Extract the python namespace from each thrift source file."""
py_namespaces_by_target = OrderedDict()
failing_py_thrift_by_target = defaultdict(list)
for t, all_content in thrift_file_sources_by_target.items():
py_namespaces_by_target[t] = []
for (path, content) in all_content:
try:
py_namespaces_by_target[t].append(
# File content is provided as a binary string, so we have to decode it.
(path,
self._extract_py_namespace_from_content(
t, path, content.decode('utf-8'))))
except self.NamespaceParseFailure:
failing_py_thrift_by_target[t].append(path)
if failing_py_thrift_by_target:
# We dump the output to a file here because the output can be very long in some repos.
no_py_namespace_output_file = os.path.join(
self.workdir, 'no-python-namespace-output.txt')
pretty_printed_failures = '\n'.join(
'{}: [{}]'.format(t.address.spec, ', '.join(paths))
for t, paths in failing_py_thrift_by_target.items())
error = self.NamespaceExtractionError(
no_py_namespace_output_file, """\
Python namespaces could not be extracted from some thrift sources. Declaring a `namespace py` in
thrift sources for python thrift library targets will soon become required.
{} python library target(s) contained thrift sources not declaring a python namespace. The targets
and/or files which need to be edited will be dumped to: {}
""".format(len(failing_py_thrift_by_target), no_py_namespace_output_file))
safe_file_dump(no_py_namespace_output_file,
'{}\n'.format(pretty_printed_failures),
mode='w')
if self.get_options().strict:
raise error
else:
self.context.log.warn(error)
return py_namespaces_by_target
def test_reduced_dependencies_diamond(self):
# bar <-- foo --> baz
# | |
# `----> bak <----'
dep_map = OrderedDict(foo=['bar', 'baz'],
bar=['bak'],
baz=['bak'],
bak=[])
target_map = self.create_dependencies(dep_map)
self.assertEqual(
self.dependency_calculator.reduced_dependencies(target_map['foo']),
OrderedSet([target_map['bar'], target_map['baz']]))
self.assertEqual(
self.dependency_calculator.reduced_dependencies(target_map['bar']),
OrderedSet([target_map['bak']]))
self.assertEqual(
self.dependency_calculator.reduced_dependencies(target_map['baz']),
OrderedSet([target_map['bak']]))
self.assert_requirements(target_map['foo'],
{'bar==0.0.0', 'baz==0.0.0'})
self.assert_requirements(target_map['bar'], {'bak==0.0.0'})
self.assert_requirements(target_map['baz'], {'bak==0.0.0'})
def create(cls, spec_path, address_maps):
"""Creates an address family from the given set of address maps.
:param spec_path: The directory prefix shared by all address_maps.
:param address_maps: The family of maps that form this namespace.
:type address_maps: :class:`collections.Iterable` of :class:`AddressMap`
:returns: a new address family.
:rtype: :class:`AddressFamily`
:raises: :class:`MappingError` if the given address maps do not form a family.
"""
if spec_path == '.':
spec_path = ''
for address_map in address_maps:
if not address_map.path.startswith(spec_path):
raise DifferingFamiliesError(
'Expected AddressMaps to share the same parent directory {}, '
'but received: {}'.format(spec_path, address_map.path))
objects_by_name = {}
for address_map in address_maps:
current_path = address_map.path
for name, obj in address_map.objects_by_name.items():
previous = objects_by_name.get(name)
if previous:
previous_path, _ = previous
raise DuplicateNameError(
'An object with name {name!r} is already defined in '
'{previous_path!r}, will not overwrite with {obj!r} from '
'{current_path!r}.'.format(name=name,
previous_path=previous_path,
obj=obj,
current_path=current_path))
objects_by_name[name] = (current_path, obj)
return AddressFamily(
namespace=spec_path,
objects_by_name=OrderedDict(
(name, (path, obj))
for name, (path, obj) in sorted(objects_by_name.items())))
def create(cls, rule_entries):
"""Creates a RuleIndex with tasks indexed by their output type."""
serializable_rules = OrderedDict()
serializable_roots = OrderedSet()
def add_task(product_type, rule):
if product_type not in serializable_rules:
serializable_rules[product_type] = OrderedSet()
serializable_rules[product_type].add(rule)
def add_rule(rule):
if isinstance(rule, RootRule):
serializable_roots.add(rule)
return
# TODO: Ensure that interior types work by indexing on the list of types in
# the constraint. This heterogenity has some confusing implications:
# see https://github.com/pantsbuild/pants/issues/4005
for kind in rule.output_constraint.types:
add_task(kind, rule)
add_task(rule.output_constraint, rule)
for entry in rule_entries:
if isinstance(entry, Rule):
add_rule(entry)
elif hasattr(entry, '__call__'):
rule = getattr(entry, 'rule', None)
if rule is None:
raise TypeError(
"Expected callable {} to be decorated with @rule.".
format(entry))
add_rule(rule)
else:
raise TypeError(
"Unexpected rule type: {}. "
"Rules either extend Rule, or are static functions "
"decorated with @rule.".format(type(entry)))
return cls(serializable_rules, serializable_roots)
def get_arg_descriptions_from_docstring(cls, obj):
"""Returns an ordered map of arg name -> arg description found in :param: stanzas."""
ret = OrderedDict()
name = ''
doc = obj.__doc__ or ''
lines = [s.strip() for s in doc.split('\n')]
stanza_first_line_re = cls._get_stanza_first_line_re()
for line in lines:
m = stanza_first_line_re.match(line)
if m and m.group(1) == 'param':
# If first line of a parameter description, set name and description.
name, description = m.group(3, 4)
ret[name] = description
elif m and m.group(1) != 'param':
# If first line of a description of an item other than a parameter, clear name.
name = ''
elif name and line:
# If subsequent line of a parameter description, add to existing description (if any) for
# that parameter.
ret[name] += (' ' + line) if ret[name] else line
# Ignore subsequent lines of descriptions of items other than parameters.
return ret
def parse(cls, filepath, filecontent, parser):
"""Parses a source for addressable Serializable objects.
No matter the parser used, the parsed and mapped addressable objects are all 'thin'; ie: any
objects they point to in other namespaces or even in the same namespace but from a seperate
source are left as unresolved pointers.
:param string filepath: The path to the byte source containing serialized objects.
:param string filecontent: The content of byte source containing serialized objects to be parsed.
:param symbol_table: The symbol table cls to expose a symbol table dict.
:type symbol_table: Instance of :class:`pants.engine.parser.SymbolTable`.
:param parser: The parser cls to use.
:type parser: A :class:`pants.engine.parser.Parser`.
"""
try:
objects = parser.parse(filepath, filecontent)
except Exception as e:
raise MappingError('Failed to parse {}:\n{}'.format(filepath, e))
objects_by_name = {}
for obj in objects:
if not Serializable.is_serializable(obj):
raise UnaddressableObjectError(
'Parsed a non-serializable object: {!r}'.format(obj))
attributes = obj._asdict()
name = attributes.get('name')
if not name:
raise UnaddressableObjectError(
'Parsed a non-addressable object: {!r}'.format(obj))
if name in objects_by_name:
raise DuplicateNameError(
'An object already exists at {!r} with name {!r}: {!r}. Cannot '
'map {!r}'.format(filepath, name, objects_by_name[name],
obj))
objects_by_name[name] = obj
return cls(filepath, OrderedDict(sorted(objects_by_name.items())))
def _topological_sort(self, goal_info_by_goal):
dependees_by_goal = OrderedDict()
def add_dependee(goal, dependee=None):
dependees = dependees_by_goal.get(goal)
if dependees is None:
dependees = set()
dependees_by_goal[goal] = dependees
if dependee:
dependees.add(dependee)
for goal, goal_info in goal_info_by_goal.items():
add_dependee(goal)
for dependency in goal_info.goal_dependencies:
add_dependee(dependency, goal)
satisfied = set()
while dependees_by_goal:
count = len(dependees_by_goal)
for goal, dependees in dependees_by_goal.items():
unsatisfied = len(dependees - satisfied)
if unsatisfied == 0:
satisfied.add(goal)
dependees_by_goal.pop(goal)
yield goal_info_by_goal[goal]
break
if len(dependees_by_goal) == count:
for dependees in dependees_by_goal.values():
dependees.difference_update(satisfied)
# TODO(John Sirois): Do a better job here and actually collect and print cycle paths
# between Goals/Tasks. The developer can most directly address that data.
raise self.GoalCycleError(
'Cycle detected in goal dependencies:\n\t{0}'.format(
'\n\t'.join(
'{0} <- {1}'.format(goal, list(dependees))
for goal, dependees in dependees_by_goal.items())))
def _run_pytest(self, fail_fast, test_targets, workdirs):
if not test_targets:
return PytestResult.rc(0)
# Absolute path to chrooted test file -> Path to original test file relative to the buildroot.
sources_map = OrderedDict()
for t in test_targets:
for p in t.sources_relative_to_source_root():
sources_map[os.path.join(self._source_chroot_path,
p)] = os.path.join(t.target_base, p)
if not sources_map:
return PytestResult.rc(0)
with self._test_runner(workdirs, test_targets,
sources_map) as (pytest_binary, test_args,
get_pytest_rootdir):
# Validate that the user didn't provide any passthru args that conflict
# with those we must set ourselves.
for arg in self.get_passthru_args():
if arg.startswith('--junitxml') or arg.startswith(
'--confcutdir'):
raise TaskError(
'Cannot pass this arg through to pytest: {}'.format(
arg))
junitxml_path = workdirs.junitxml_path(*test_targets)
# N.B. the `--confcutdir` here instructs pytest to stop scanning for conftest.py files at the
# top of the buildroot. This prevents conftest.py files from outside (e.g. in users home dirs)
# from leaking into pants test runs. See: https://github.com/pantsbuild/pants/issues/2726
args = [
'-c', pytest_binary.config_path, '--junitxml', junitxml_path,
'--confcutdir',
get_buildroot(), '--continue-on-collection-errors'
]
if fail_fast:
args.extend(['-x'])
if self._debug:
args.extend(['-s'])
if self.get_options().colors:
args.extend(['--color', 'yes'])
if self.get_options().options:
for opt in self.get_options().options:
args.extend(safe_shlex_split(opt))
args.extend(self.get_passthru_args())
args.extend(test_args)
args.extend(sources_map.keys())
# We want to ensure our reporting based off junit xml is from this run so kill results from
# prior runs.
if os.path.exists(junitxml_path):
os.unlink(junitxml_path)
with self._maybe_run_in_chroot():
result = self._do_run_tests_with_args(pytest_binary.pex, args)
# There was a problem prior to test execution preventing junit xml file creation so just let
# the failure result bubble.
if not os.path.exists(junitxml_path):
return result
pytest_rootdir = get_pytest_rootdir()
failed_targets = self._get_failed_targets_from_junitxml(
junitxml_path, test_targets, pytest_rootdir)
def parse_error_handler(parse_error):
# Simple error handler to pass to xml parsing function.
raise TaskError('Error parsing xml file at {}: {}'.format(
parse_error.xml_path, parse_error.cause))
all_tests_info = self.parse_test_info(
junitxml_path, parse_error_handler,
['file', 'name', 'classname'])
for test_name, test_info in all_tests_info.items():
test_target = self._get_target_from_test(
test_info, test_targets, pytest_rootdir)
self.report_all_info_for_single_test(self.options_scope,
test_target, test_name,
test_info)
return result.with_failed_targets(failed_targets)
def datatype(field_decls, superclass_name=None, **kwargs):
"""A wrapper for `namedtuple` that accounts for the type of the object in equality.
Field declarations can be a string, which declares a field with that name and
no type checking. Field declarations can also be a tuple `('field_name',
field_type)`, which declares a field named `field_name` which is type-checked
at construction. If a type is given, the value provided to the constructor for
that field must be exactly that type (i.e. `type(x) == field_type`), and not
e.g. a subclass.
:param field_decls: Iterable of field declarations.
:return: A type object which can then be subclassed.
:raises: :class:`TypeError`
"""
field_names = []
fields_with_constraints = OrderedDict()
for maybe_decl in field_decls:
# ('field_name', type)
if isinstance(maybe_decl, tuple):
field_name, type_spec = maybe_decl
if isinstance(type_spec, type):
type_constraint = Exactly(type_spec)
elif isinstance(type_spec, TypeConstraint):
type_constraint = type_spec
else:
raise TypeError(
"type spec for field '{}' was not a type or TypeConstraint: was {!r} (type {!r})."
.format(field_name, type_spec,
type(type_spec).__name__))
fields_with_constraints[field_name] = type_constraint
else:
# interpret it as a field name without a type to check
field_name = maybe_decl
# namedtuple() already checks field uniqueness
field_names.append(field_name)
if not superclass_name:
superclass_name = '_anonymous_namedtuple_subclass'
namedtuple_cls = namedtuple(superclass_name, field_names, **kwargs)
class DataType(namedtuple_cls, DatatypeMixin):
type_check_error_type = TypedDatatypeInstanceConstructionError
def __new__(cls, *args, **kwargs):
# TODO: Ideally we could execute this exactly once per `cls` but it should be a
# relatively cheap check.
if not hasattr(cls.__eq__, '_eq_override_canary'):
raise cls.make_type_error('Should not override __eq__.')
try:
this_object = super(DataType,
cls).__new__(cls, *args, **kwargs)
except TypeError as e:
raise cls.make_type_error(
"error in namedtuple() base constructor: {}".format(e))
# TODO: Make this kind of exception pattern (filter for errors then display them all at once)
# more ergonomic.
type_failure_msgs = []
for field_name, field_constraint in fields_with_constraints.items(
):
field_value = getattr(this_object, field_name)
try:
field_constraint.validate_satisfied_by(field_value)
except TypeConstraintError as e:
type_failure_msgs.append(
"field '{}' was invalid: {}".format(field_name, e))
if type_failure_msgs:
raise cls.make_type_error(
'error(s) type checking constructor arguments:\n{}'.format(
'\n'.join(type_failure_msgs)))
return this_object
def __eq__(self, other):
if self is other:
return True
# Compare types and fields.
if type(self) != type(other):
return False
# Explicitly return super.__eq__'s value in case super returns NotImplemented
return super(DataType, self).__eq__(other)
# We define an attribute on the `cls` level definition of `__eq__` that will allow us to detect
# that it has been overridden.
__eq__._eq_override_canary = None
def __ne__(self, other):
return not (self == other)
# NB: in Python 3, whenever __eq__ is overridden, __hash__() must also be
# explicitly implemented, otherwise Python will raise "unhashable type". See
# https://docs.python.org/3/reference/datamodel.html#object.__hash__.
def __hash__(self):
return super(DataType, self).__hash__()
# NB: As datatype is not iterable, we need to override both __iter__ and all of the
# namedtuple methods that expect self to be iterable.
def __iter__(self):
raise self.make_type_error("datatype object is not iterable")
def _super_iter(self):
return super(DataType, self).__iter__()
def _asdict(self):
"""Return a new OrderedDict which maps field names to their values.
Overrides a namedtuple() method which calls __iter__.
"""
return OrderedDict(zip(self._fields, self._super_iter()))
def _replace(self, **kwargs):
"""Return a new datatype object replacing specified fields with new values.
Overrides a namedtuple() method which calls __iter__.
"""
field_dict = self._asdict()
field_dict.update(**kwargs)
return type(self)(**field_dict)
def copy(self, **kwargs):
return self._replace(**kwargs)
# NB: it is *not* recommended to rely on the ordering of the tuple returned by this method.
def __getnewargs__(self):
"""Return self as a plain tuple. Used by copy and pickle."""
return tuple(self._super_iter())
def __repr__(self):
args_formatted = []
for field_name in field_names:
field_value = getattr(self, field_name)
args_formatted.append("{}={!r}".format(field_name,
field_value))
return '{class_name}({args_joined})'.format(
class_name=type(self).__name__,
args_joined=', '.join(args_formatted))
def __str__(self):
elements_formatted = []
for field_name in field_names:
constraint_for_field = fields_with_constraints.get(
field_name, None)
field_value = getattr(self, field_name)
if not constraint_for_field:
elements_formatted.append(
# TODO: consider using the repr of arguments in this method.
"{field_name}={field_value}".format(
field_name=field_name, field_value=field_value))
else:
elements_formatted.append(
"{field_name}<{type_constraint}>={field_value}".format(
field_name=field_name,
type_constraint=constraint_for_field,
field_value=field_value))
return '{class_name}({typed_tagged_elements})'.format(
class_name=type(self).__name__,
typed_tagged_elements=', '.join(elements_formatted))
# Return a new type with the given name, inheriting from the DataType class
# just defined, with an empty class body.
try: # Python3
return type(superclass_name, (DataType, ), {})
except TypeError: # Python2
return type(superclass_name.encode('utf-8'), (DataType, ), {})
def sort_goals(self, context, goals):
goal_info_by_goal = OrderedDict()
for goal in reversed(OrderedSet(goals)):
self._visit_goal(goal, context, goal_info_by_goal)
return list(reversed(list(self._topological_sort(goal_info_by_goal))))