def execution_request(self, products, subjects):
"""Create and return an ExecutionRequest for the given products and subjects.
The resulting ExecutionRequest object will contain keys tied to this scheduler's product Graph, and
so it will not be directly usable with other scheduler instances without being re-created.
An ExecutionRequest for an Address represents exactly one product output, as does SingleAddress. But
we differentiate between them here in order to normalize the output for all Spec objects
as "list of product".
:param products: A list of product types to request for the roots.
:type products: list of types
:param subjects: A list of Spec and/or PathGlobs objects.
:type subject: list of :class:`pants.base.specs.Spec`, `pants.build_graph.Address`, and/or
:class:`pants.engine.fs.PathGlobs` objects.
:returns: An ExecutionRequest for the given products and subjects.
"""
return ExecutionRequest(tuple((s, Select(p)) for s in subjects for p in products))
def test_noop_removal_full_single_subject_type(self):
rules = _suba_root_rules + [
TaskRule(Exactly(A), [Select(C)], noop),
TaskRule(Exactly(A), [], noop),
]
fullgraph = self.create_full_graph(RuleIndex.create(rules))
self.assert_equal_with_printing(
dedent("""
digraph {
// root subject types: SubA
// root entries
"Select(A) for SubA" [color=blue]
"Select(A) for SubA" -> {"(A, (,), noop) of SubA"}
// internal entries
"(A, (,), noop) of SubA" -> {}
}""").strip(), fullgraph)
def create_graph_tasks(address_mapper, symbol_table_cls):
"""Creates tasks used to parse Structs from BUILD files.
:param address_mapper_key: The subject key for an AddressMapper instance.
:param symbol_table_cls: A SymbolTable class to provide symbols for Address lookups.
"""
return [
# Support for resolving Structs from Addresses
(Struct, [
Select(UnhydratedStruct),
SelectDependencies(Struct, UnhydratedStruct)
], hydrate_struct),
(UnhydratedStruct, [
SelectProjection(AddressFamily, Dir, ('spec_path', ), Address),
Select(Address)
], resolve_unhydrated_struct),
] + [
# BUILD file parsing.
(AddressFamily, [
SelectLiteral(address_mapper, AddressMapper),
Select(Dir),
SelectProjection(FilesContent, Files, ('files', ), BuildFiles)
], parse_address_family),
(BuildFiles,
[SelectLiteral(address_mapper, AddressMapper),
Select(Files)], filter_buildfile_paths),
] + [
# Addresses for user-defined products might possibly be resolvable from BLD files. These tasks
# define that lookup for each literal product.
(product, [Select(Struct)], identity)
for product in symbol_table_cls.table().values()
] + [
# Spec handling.
(Addresses, [
SelectProjection(AddressFamily, Dir,
('directory', ), SingleAddress),
Select(SingleAddress)
], address_from_address_family),
(Addresses, [
SelectProjection(AddressFamily, Dir,
('directory', ), SiblingAddresses)
], addresses_from_address_family),
(Addresses, [SelectDependencies(AddressFamily, Dirs)
], addresses_from_address_families),
(PathGlobs, [Select(DescendantAddresses)
], descendant_addresses_to_globs),
]
def test_failed_output_conversion_propagates_throw(self):
scheduler = self.mk_scheduler_in_example_fs([
# subject to files / product of subject to files for snapshot.
SnapshottedProcess.create(product_type=Concatted,
binary_type=ShellCatToOutFile,
input_selectors=(Select(Snapshot),),
input_conversion=file_list_to_args_for_cat_with_snapshot_subjects_and_output_file,
output_conversion=fail_process_result),
SingletonRule(ShellCatToOutFile, ShellCatToOutFile()),
])
request = scheduler.execution_request([Concatted],
[PathGlobs.create('', include=['fs_test/a/b/*'])])
root_entries = scheduler.execute(request).root_products
self.assertEquals(1, len(root_entries))
self.assertFirstEntryIsThrow(root_entries,
in_msg='Failed in output conversion!')
def test_ruleset_with_failure_due_to_incompatible_subject_for_singleton(self):
rules = [
RootRule(A),
TaskRule(D, [Select(C)], noop),
SingletonRule(B, B()),
]
validator = self.create_validator({}, rules)
with self.assertRaises(ValueError) as cm:
validator.assert_ruleset_valid()
# This error message could note near matches like the singleton.
self.assert_equal_with_printing(dedent("""
Rules with errors: 1
(D, (Select(C),), noop):
no matches for Select(C) with subject types: A
""").strip(),
str(cm.exception))
def test_ruleset_with_failure_due_to_incompatible_subject_for_singleton(
self):
rules = [
RootRule(A),
TaskRule(D, [Select(C)], noop),
SingletonRule(B, B()),
]
with self.assertRaises(Exception) as cm:
create_scheduler(rules)
# This error message could note near matches like the singleton.
self.assert_equal_with_printing(
dedent("""
Rules with errors: 1
(D, [Select(C)], noop):
no rule was available to compute C with parameter type A
""").strip(), str(cm.exception))
def test_select_projection_simple(self):
rules = [
(Exactly(A), (SelectProjection(B, D, ('some',), SubA),), noop),
(B, (Select(D),), noop),
]
graphmaker = GraphMaker(NodeBuilder.create(rules, tuple()),
root_subject_fns=_suba_root_subject_fns)
subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A)
self.assert_equal_with_printing(dedent("""
{
root_subject_types: (SubA,)
root_rules: (Exactly(A), (SelectProjection(B, D, (u'some',), SubA),), noop) of SubA
(B, (Select(D),), noop) of D => (SubjectIsProduct(D),)
(Exactly(A), (SelectProjection(B, D, (u'some',), SubA),), noop) of SubA => (SubjectIsProduct(SubA), (B, (Select(D),), noop) of D,)
}""").strip(),
subgraph)
def test_get_with_matching_singleton(self):
rules = [
TaskRule(Exactly(A), [Select(SubA)], noop, input_gets=[Get(B, C)]),
SingletonRule(B, B()),
]
subgraph = self.create_subgraph(A, rules, SubA())
self.assert_equal_with_printing(
dedent("""
digraph {
// root subject types: SubA
// root entries
"Select(A) for SubA" [color=blue]
"Select(A) for SubA" -> {"(A, [Select(SubA)], [Get(B, C)], noop) for SubA"}
// internal entries
"(A, [Select(SubA)], [Get(B, C)], noop) for SubA" -> {"Param(SubA)" "Singleton(B(), B)"}
}""").strip(), subgraph)
def test_select_dependencies_with_subject_as_first_subselector(self):
rules = [
(Exactly(A), (SelectDependencies(B, SubA, field_types=(D,)),), noop),
(B, (Select(D),), noop),
]
graphmaker = GraphMaker(NodeBuilder.create(rules, tuple()),
root_subject_fns=_suba_root_subject_fns)
subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A)
self.assert_equal_with_printing(dedent("""
{
root_subject_types: (SubA,)
root_rules: (Exactly(A), (SelectDependencies(B, SubA, u'dependencies', field_types=(D,)),), noop) of SubA
(B, (Select(D),), noop) of D => (SubjectIsProduct(D),)
(Exactly(A), (SelectDependencies(B, SubA, u'dependencies', field_types=(D,)),), noop) of SubA => (SubjectIsProduct(SubA), (B, (Select(D),), noop) of D,)
}""").strip(),
subgraph)
def create_legacy_graph_tasks(symbol_table_cls):
"""Create tasks to recursively parse the legacy graph."""
symbol_table_constraint = symbol_table_cls.constraint()
return [
transitive_hydrated_targets,
TaskRule(HydratedTarget, [
Select(symbol_table_constraint),
SelectDependencies(HydratedField,
symbol_table_constraint,
'field_adaptors',
field_types=(
SourcesField,
BundlesField,
))
], hydrate_target),
hydrate_sources,
hydrate_bundles,
]
def __init__(self,
goals,
tasks,
project_tree,
graph_lock=None,
inline_nodes=True,
graph_validator=None):
"""
:param goals: A dict from a goal name to a product type. A goal is just an alias for a
particular (possibly synthetic) product.
:param tasks: A set of (output, input selection clause, task function) triples which
is used to compute values in the product graph.
:param project_tree: An instance of ProjectTree for the current build root.
:param graph_lock: A re-entrant lock to use for guarding access to the internal ProductGraph
instance. Defaults to creating a new threading.RLock().
:param inline_nodes: Whether to inline execution of `inlineable` Nodes. This improves
performance, but can make debugging more difficult because the entire
execution history is not recorded in the ProductGraph.
:param graph_validator: A validator that runs over the entire graph after every scheduling
attempt. Very expensive, very experimental.
"""
self._products_by_goal = goals
self._project_tree = project_tree
self._node_builder = NodeBuilder.create(tasks)
self._graph_validator = graph_validator
self._product_graph = ProductGraph()
self._product_graph_lock = graph_lock or threading.RLock()
self._inline_nodes = inline_nodes
select_product = lambda product: Select(product)
select_dep_addrs = lambda product: SelectDependencies(
product, Addresses, field_types=(Address, ))
self._root_selector_fns = {
Address: select_product,
PathGlobs: select_product,
SingleAddress: select_dep_addrs,
SiblingAddresses: select_dep_addrs,
AscendantAddresses: select_dep_addrs,
DescendantAddresses: select_dep_addrs,
}
RulesetValidator(self._node_builder, goals,
self._root_selector_fns.keys()).validate()
def test_select_projection_simple(self):
rules = [
TaskRule(Exactly(A), [SelectProjection(B, D, 'some', SubA)], noop),
TaskRule(B, [Select(D)], noop),
]
subgraph = self.create_subgraph(A, rules, SubA())
self.assert_equal_with_printing(
dedent("""
digraph {
// root subject types: SubA
// root entries
"Select(A) for SubA" [color=blue]
"Select(A) for SubA" -> {"(A, (SelectProjection(B, D, 'some', SubA),), noop) of SubA"}
// internal entries
"(A, (SelectProjection(B, D, 'some', SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)" "(B, (Select(D),), noop) of D"}
"(B, (Select(D),), noop) of D" -> {"SubjectIsProduct(D)"}
}""").strip(), subgraph)
def test_get_simple(self):
rules = [
TaskRule(Exactly(A), [], noop, [Get(B, D)]),
TaskRule(B, [Select(D)], noop),
]
subgraph = self.create_subgraph(A, rules, SubA())
self.assert_equal_with_printing(
dedent("""
digraph {
// root subject types: SubA
// root entries
"Select(A) for ()" [color=blue]
"Select(A) for ()" -> {"(A, [], [Get(B, D)], noop) for ()"}
// internal entries
"(A, [], [Get(B, D)], noop) for ()" -> {"(B, [Select(D)], noop) for D"}
"(B, [Select(D)], noop) for D" -> {"Param(D)"}
}""").strip(), subgraph)
def test_get_with_matching_singleton(self):
rules = [
TaskRule(Exactly(A), [Select(SubA)], noop, input_gets=[Get(B, C)]),
SingletonRule(B, B()),
]
subgraph = self.create_subgraph(A, rules, SubA())
#TODO perhaps singletons should be marked in the dot format somehow
self.assert_equal_with_printing(dedent("""
digraph {
// root subject types: SubA
// root entries
"Select(A) for SubA" [color=blue]
"Select(A) for SubA" -> {"(A, (Select(SubA),), [Get(B, C)], noop) of SubA"}
// internal entries
"(A, (Select(SubA),), [Get(B, C)], noop) of SubA" -> {"SubjectIsProduct(SubA)" "Singleton(B(), B)"}
}""").strip(),
subgraph)
def test_ruleset_with_failure_due_to_incompatible_subject_for_intrinsic(self):
rules = [
(D, (Select(C),), noop)
]
intrinsics = [
(B, C, noop),
]
validator = self.create_validator({}, intrinsics, {A}, rules)
with self.assertRaises(ValueError) as cm:
validator.assert_ruleset_valid()
# This error message could note near matches like the intrinsic.
self.assert_equal_with_printing(dedent("""
Rules with errors: 1
(D, (Select(C),), noop):
no matches for Select(C) with subject types: A
""").strip(),
str(cm.exception))
def test_noop_removal_in_subgraph(self):
rules = [
TaskRule(Exactly(A), [Select(C)], noop),
TaskRule(Exactly(A), [], noop),
SingletonRule(B, B()),
]
subgraph = self.create_subgraph(A, rules, SubA(), validate=False)
self.assert_equal_with_printing(
dedent("""
digraph {
// root subject types: SubA
// root entries
"Select(A) for ()" [color=blue]
"Select(A) for ()" -> {"(A, [], noop) for ()"}
// internal entries
"(A, [], noop) for ()" -> {}
}""").strip(), subgraph)
def test_javac_version_example(self):
sources = PathGlobs.create(
'', include=['inputs/src/java/simple/Simple.java'])
scheduler = self.mk_scheduler_in_example_fs([
ExecuteProcess.create_in(
product_type=ExecuteProcessRequest,
input_selectors=(Select(Javac), ),
input_conversion=process_request_from_java_sources),
SingletonRule(Javac, Javac()),
])
req = scheduler.product_request(ExecuteProcessRequest, [sources])
request = scheduler.execution_request([ExecuteProcessResult], req)
root_entries = scheduler.execute(request).root_products
self.assertEquals(1, len(root_entries))
state = self.assertFirstEntryIsReturn(root_entries, scheduler)
result = state.value
self.assertEqual(0, result.exit_code)
self.assertIn('javac', result.stderr)
def test_select_dependencies_with_subject_as_first_subselector(self):
rules = [
TaskRule(Exactly(A), [SelectDependencies(B, SubA, field_types=(D,))], noop),
TaskRule(B, [Select(D)], noop),
]
subgraph = self.create_subgraph(A, rules, SubA())
self.assert_equal_with_printing(dedent("""
digraph {
// root subject types: SubA
// root entries
"Select(A) for SubA" [color=blue]
"Select(A) for SubA" -> {"(A, (SelectDependencies(B, SubA, field_types=(D,)),), noop) of SubA"}
// internal entries
"(A, (SelectDependencies(B, SubA, field_types=(D,)),), noop) of SubA" -> {"SubjectIsProduct(SubA)" "(B, (Select(D),), noop) of D"}
"(B, (Select(D),), noop) of D" -> {"SubjectIsProduct(D)"}
}""").strip(),
subgraph)
def test_integration_concat_with_snapshot_subjects_test(self):
scheduler = self.mk_scheduler_in_example_fs([
# subject to files / product of subject to files for snapshot.
SnapshottedProcess.create(product_type=Concatted,
binary_type=ShellCatToOutFile,
input_selectors=(Select(Snapshot),),
input_conversion=file_list_to_args_for_cat_with_snapshot_subjects_and_output_file,
output_conversion=process_result_to_concatted_from_outfile),
SingletonRule(ShellCatToOutFile, ShellCatToOutFile()),
])
request = scheduler.execution_request([Concatted],
[PathGlobs.create('', include=['fs_test/a/b/*'])])
root_entries = scheduler.execute(request).root_products
self.assertEquals(1, len(root_entries))
state = self.assertFirstEntryIsReturn(root_entries, scheduler, request)
concatted = state.value
self.assertEqual(Concatted('one\ntwo\n'), concatted)
def test_noop_removal_full_single_subject_type(self):
intrinsics = {(B, C): BoringRule(C)}
rules = [
# C is provided by an intrinsic, but only if the subject is B.
(Exactly(A), (Select(C),), noop),
(Exactly(A), tuple(), noop),
]
graphmaker = GraphMaker(NodeBuilder.create(rules,
intrinsic_providers=(IntrinsicProvider(intrinsics),)),
root_subject_fns=_suba_root_subject_fns)
fullgraph = graphmaker.full_graph()
self.assert_equal_with_printing(dedent("""
{
root_subject_types: (SubA,)
root_rules: (Exactly(A), (), noop) of SubA
(Exactly(A), (), noop) of SubA => (,)
}""").strip(), fullgraph)
def test_select_dependencies_multiple_field_types_all_resolvable(self):
rules = [
(Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop),
(B, (Select(Exactly(C, D)),), noop),
]
subgraph = self.create_subgraph(A, rules, SubA())
self.assert_equal_with_printing(dedent("""
digraph {
// root subject types: SubA
// root entries
"Select(A) for SubA" [color=blue]
"Select(A) for SubA" -> {"(A, (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA"}
// internal entries
"(A, (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA" -> {"SubjectIsProduct(SubA)" "(B, (Select(Exactly(C, D)),), noop) of C" "(B, (Select(Exactly(C, D)),), noop) of D"}
"(B, (Select(Exactly(C, D)),), noop) of C" -> {"SubjectIsProduct(C)"}
"(B, (Select(Exactly(C, D)),), noop) of D" -> {"SubjectIsProduct(D)"}
}""").strip(),
subgraph)
def test_ruleset_unreachable_due_to_product_of_select_dependencies(self):
rules = [
(A, (SelectDependencies(B, SubA, field_types=(D,)),), noop),
]
intrinsics = [
(B, C, noop),
]
validator = RulesetValidator(NodeBuilder.create(rules, intrinsics),
goal_to_product={},
root_subject_fns={k: lambda p: Select(p) for k in (A,)})
with self.assertRaises(ValueError) as cm:
validator.validate()
self.assert_equal_with_printing(dedent("""
Rules with errors: 1
(A, (SelectDependencies(B, SubA, u'dependencies', field_types=(D,)),), noop):
Unreachable with subject types: Any
""").strip(),
str(cm.exception))
def create_legacy_graph_tasks(symbol_table):
"""Create tasks to recursively parse the legacy graph."""
symbol_table_constraint = symbol_table.constraint()
return [
transitive_hydrated_targets,
transitive_hydrated_target,
hydrated_targets,
TaskRule(
HydratedTarget,
[Select(symbol_table_constraint)],
hydrate_target,
input_gets=[
Get(HydratedField, SourcesField),
Get(HydratedField, BundlesField),
]
),
hydrate_sources,
hydrate_bundles,
]
class SnapshotIntrinsicRule(Rule):
"""Intrinsic rule for snapshot process execution."""
output_product_type = Snapshot
input_selectors = (Select(Files), )
def as_node(self, subject, variants):
assert type(subject) in (Files, PathGlobs)
return SnapshotNode.create(subject, variants)
@classmethod
def as_intrinsics(cls):
snapshot_intrinsic_rule = cls()
return {
(Files, Snapshot): snapshot_intrinsic_rule,
(PathGlobs, Snapshot): snapshot_intrinsic_rule
}
def __repr__(self):
return '{}()'.format(type(self).__name__)
def test_smallest_full_test(self):
@rule(Exactly(A), [Select(SubA)])
def a_from_suba(suba):
pass
rules = _suba_root_rules + [
RootRule(SubA),
a_from_suba,
]
fullgraph = self.create_full_graph(rules)
self.assert_equal_with_printing(
dedent("""
digraph {
// root subject types: SubA
// root entries
"Select(A) for SubA" [color=blue]
"Select(A) for SubA" -> {"(A, [Select(SubA)], a_from_suba) for SubA"}
// internal entries
"(A, [Select(SubA)], a_from_suba) for SubA" -> {"Param(SubA)"}
}""").strip(), fullgraph)
def test_noop_removal_in_subgraph(self):
rules = [
# C is provided by an intrinsic, but only if the subject is B.
(Exactly(A), (Select(C),), noop),
(Exactly(A), tuple(), noop),
]
intrinsics = [
(B, C, noop),
]
graphmaker = GraphMaker(NodeBuilder.create(rules,
intrinsics),
root_subject_fns=_suba_root_subject_fns)
subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A)
self.assert_equal_with_printing(dedent("""
{
root_subject_types: (SubA,)
root_rules: (Exactly(A), (), noop) of SubA
(Exactly(A), (), noop) of SubA => (,)
}""").strip(), subgraph)
def test_integration_concat_with_snapshot_subjects_test(self):
scheduler = self.mk_scheduler_in_example_fs([
# subject to files / product of subject to files for snapshot.
SnapshottedProcess(product_type=Concatted,
binary_type=ShellCatToOutFile,
input_selectors=(Select(Files),),
input_conversion=file_list_to_args_for_cat_with_snapshot_subjects_and_output_file,
output_conversion=process_result_to_concatted_from_outfile),
[ShellCatToOutFile, [], ShellCatToOutFile]
])
request = scheduler.execution_request([Concatted],
[PathGlobs.create('', globs=['fs_test/a/b/*'])])
LocalSerialEngine(scheduler).reduce(request)
root_entries = scheduler.root_entries(request).items()
self.assertEquals(1, len(root_entries))
state = self.assertFirstEntryIsReturn(root_entries, scheduler)
concatted = state.value
self.assertEqual(Concatted('one\ntwo\n'), concatted)
def test_single_rule_depending_on_subject_selection(self):
@rule(Exactly(A), [Select(SubA)])
def a_from_suba(suba):
pass
rules = [
a_from_suba,
]
subgraph = self.create_subgraph(A, rules, SubA())
self.assert_equal_with_printing(
dedent("""
digraph {
// root subject types: SubA
// root entries
"Select(A) for SubA" [color=blue]
"Select(A) for SubA" -> {"(A, [Select(SubA)], a_from_suba) for SubA"}
// internal entries
"(A, [Select(SubA)], a_from_suba) for SubA" -> {"Param(SubA)"}
}""").strip(), subgraph)
def test_gen(self):
build_request = self.request([GenGoal], self.thrift)
root, = self.build(build_request)
# Root: expect the synthetic GenGoal product.
self.assert_root(
root, self.thrift,
GenGoal("non-empty input to satisfy the Goal constructor"))
variants = {'thrift': 'apache_java'}
# Expect ThriftSources to have been selected.
self.assert_select_for_subjects(walk,
Select(ThriftSources), [self.thrift],
variants=variants)
# Expect an ApacheThriftJavaConfiguration to have been used via the default Variants.
self.assert_select_for_subjects(walk,
SelectVariant(
ApacheThriftJavaConfiguration,
variant_key='thrift'),
[self.thrift],
variants=variants)
def test_include_trace_error_raises_error_with_trace(self):
rules = [RootRule(B), TaskRule(A, [Select(B)], nested_raise)]
scheduler = self.scheduler(rules, include_trace_on_error=True)
with self.assertRaises(Exception) as cm:
list(scheduler.product_request(A, subjects=[(B())]))
self.assert_equal_with_printing(
dedent('''
Received unexpected Throw state(s):
Computing Select(<pants_test.engine.test_engine.B object at 0xEEEEEEEEE>, =A)
Computing Task(<function nested_raise at 0xEEEEEEEEE>, <pants_test.engine.test_engine.B object at 0xEEEEEEEEE>, =A)
Throw(An exception for B)
Traceback (most recent call last):
File LOCATION-INFO, in extern_invoke_runnable
val = runnable(*args)
File LOCATION-INFO, in nested_raise
fn_raises(x)
File LOCATION-INFO, in fn_raises
raise Exception('An exception for {}'.format(type(x).__name__))
Exception: An exception for B
''').lstrip() + '\n', remove_locations_from_traceback(str(cm.exception)))
