def test_descendant_specs(self):
"""Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory."""
spec = self.spec_parser.parse_spec('3rdparty/jvm::')
selector = SelectDependencies(Address,
Addresses,
field_types=(Address, ))
build_request = self.scheduler.selection_request([(selector, spec)])
((subject, _), root), = self.build(build_request)
# Validate the root.
self.assertEqual(spec, subject)
self.assertEqual(tuple, type(root.value))
# Confirm that a few expected addresses are in the list.
self.assertIn(self.guava, root.value)
self.assertIn(self.managed_guava, root.value)
self.assertIn(self.managed_resolve_latest, root.value)
def test_sibling_specs(self):
"""Test that sibling Addresses are parsed in the 3rdparty/jvm directory."""
spec = self.spec_parser.parse_spec('3rdparty/jvm:')
selector = SelectDependencies(Address,
Addresses,
field_types=(Address, ))
build_request = self.scheduler.selection_request([(selector, spec)])
((subject, _), root), = self.build(build_request)
# Validate the root.
self.assertEqual(spec, subject)
self.assertEqual(tuple, type(root.value))
# Confirm that an expected address is in the list.
self.assertIn(self.guava, root.value)
# And that a subdirectory address is not.
self.assertNotIn(self.managed_guava, root.value)
def scan_build_files(self, base_path):
subject = DescendantAddresses(base_path)
selector = SelectDependencies(BuildFiles,
BuildDirs,
field_types=(Dir, ))
request = self._scheduler.selection_request([(selector, subject)])
result = self._engine.execute(request)
if result.error:
raise result.error
build_files_set = set()
for state in result.root_products.values():
for build_files in state.value:
build_files_set.update(f.path for f in build_files.files)
return build_files_set
def create_legacy_graph_tasks(symbol_table_cls):
"""Create tasks to recursively parse the legacy graph."""
symbol_table_constraint = symbol_table_cls.constraint()
return [
transitive_hydrated_targets,
TaskRule(
HydratedTarget,
[Select(symbol_table_constraint),
SelectDependencies(HydratedField,
symbol_table_constraint,
'field_adaptors',
field_types=(SourcesField, BundlesField,))],
hydrate_target
),
hydrate_sources,
hydrate_bundles,
]
def test_select_dependencies_non_matching_subselector_because_of_intrinsic(self):
rules = [
(Exactly(A), (SelectDependencies(B, SubA, field_types=(D,)),), noop),
]
intrinsics = [
(C, B, noop),
]
graphmaker = GraphMaker(NodeBuilder.create(rules, intrinsics),
root_subject_fns=_suba_root_subject_fns)
subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A)
self.assert_equal_with_printing('{empty graph}', subgraph)
self.assert_equal_with_printing(dedent("""
Rules with errors: 1
(Exactly(A), (SelectDependencies(B, SubA, u'dependencies', field_types=(D,)),), noop):
no matches for Select(B) when resolving SelectDependencies(B, SubA, u'dependencies', field_types=(D,)) with subject types: D""").strip(),
subgraph.error_message())
def test_ruleset_unreachable_due_to_product_of_select_dependencies(self):
rules = [
RootRule(A),
TaskRule(A, [SelectDependencies(B, SubA, field_types=(D, ))],
noop),
]
validator = self.create_validator({}, rules)
with self.assertRaises(ValueError) as cm:
validator.assert_ruleset_valid()
self.assert_equal_with_printing(
dedent("""
Rules with errors: 1
(A, (SelectDependencies(B, SubA, field_types=(D,)),), noop):
Unreachable with subject types: Any
""").strip(), str(cm.exception))
def test_select_dependencies_with_subject_as_first_subselector(self):
rules = [
(Exactly(A), (SelectDependencies(B, SubA, field_types=(D,)),), noop),
(B, (Select(D),), noop),
]
graphmaker = GraphMaker(NodeBuilder.create(rules, tuple()),
root_subject_fns=_suba_root_subject_fns)
subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A)
self.assert_equal_with_printing(dedent("""
{
root_subject_types: (SubA,)
root_rules: (Exactly(A), (SelectDependencies(B, SubA, u'dependencies', field_types=(D,)),), noop) of SubA
(B, (Select(D),), noop) of D => (SubjectIsProduct(D),)
(Exactly(A), (SelectDependencies(B, SubA, u'dependencies', field_types=(D,)),), noop) of SubA => (SubjectIsProduct(SubA), (B, (Select(D),), noop) of D,)
}""").strip(),
subgraph)
def __init__(self,
goals,
tasks,
project_tree,
graph_lock=None,
inline_nodes=True,
graph_validator=None):
"""
:param goals: A dict from a goal name to a product type. A goal is just an alias for a
particular (possibly synthetic) product.
:param tasks: A set of (output, input selection clause, task function) triples which
is used to compute values in the product graph.
:param project_tree: An instance of ProjectTree for the current build root.
:param graph_lock: A re-entrant lock to use for guarding access to the internal ProductGraph
instance. Defaults to creating a new threading.RLock().
:param inline_nodes: Whether to inline execution of `inlineable` Nodes. This improves
performance, but can make debugging more difficult because the entire
execution history is not recorded in the ProductGraph.
:param graph_validator: A validator that runs over the entire graph after every scheduling
attempt. Very expensive, very experimental.
"""
self._products_by_goal = goals
self._project_tree = project_tree
self._node_builder = NodeBuilder.create(tasks)
self._graph_validator = graph_validator
self._product_graph = ProductGraph()
self._product_graph_lock = graph_lock or threading.RLock()
self._inline_nodes = inline_nodes
select_product = lambda product: Select(product)
select_dep_addrs = lambda product: SelectDependencies(
product, Addresses, field_types=(Address, ))
self._root_selector_fns = {
Address: select_product,
PathGlobs: select_product,
SingleAddress: select_dep_addrs,
SiblingAddresses: select_dep_addrs,
AscendantAddresses: select_dep_addrs,
DescendantAddresses: select_dep_addrs,
}
RulesetValidator(self._node_builder, goals,
self._root_selector_fns.keys()).validate()
def test_select_dependencies_with_matching_singleton(self):
rules = [
TaskRule(Exactly(A), [SelectDependencies(B, SubA, field_types=(C,))], noop),
SingletonRule(B, B()),
]
subgraph = self.create_subgraph(A, rules, SubA())
#TODO perhaps singletons should be marked in the dot format somehow
self.assert_equal_with_printing(dedent("""
digraph {
// root subject types: SubA
// root entries
"Select(A) for SubA" [color=blue]
"Select(A) for SubA" -> {"(A, (SelectDependencies(B, SubA, field_types=(C,)),), noop) of SubA"}
// internal entries
"(A, (SelectDependencies(B, SubA, field_types=(C,)),), noop) of SubA" -> {"SubjectIsProduct(SubA)" "Singleton(B(), B)"}
}""").strip(),
subgraph)
def test_select_dependencies_with_subject_as_first_subselector(self):
rules = [
TaskRule(Exactly(A), [SelectDependencies(B, SubA, field_types=(D,))], noop),
TaskRule(B, [Select(D)], noop),
]
subgraph = self.create_subgraph(A, rules, SubA())
self.assert_equal_with_printing(dedent("""
digraph {
// root subject types: SubA
// root entries
"Select(A) for SubA" [color=blue]
"Select(A) for SubA" -> {"(A, (SelectDependencies(B, SubA, field_types=(D,)),), noop) of SubA"}
// internal entries
"(A, (SelectDependencies(B, SubA, field_types=(D,)),), noop) of SubA" -> {"SubjectIsProduct(SubA)" "(B, (Select(D),), noop) of D"}
"(B, (Select(D),), noop) of D" -> {"SubjectIsProduct(D)"}
}""").strip(),
subgraph)
def test_successful_when_one_field_type_is_unfulfillable(self):
# NB We may want this to be a warning, since it may not be intentional
rules = [
(B, (Select(SubA),), noop),
(D, (Select(Exactly(B)), SelectDependencies(B, SubA, field_types=(SubA, C))), noop)
]
graphmaker = GraphMaker(NodeBuilder.create(rules, tuple()),
root_subject_fns=_suba_root_subject_fns)
subgraph = graphmaker.generate_subgraph(SubA(), requested_product=D)
self.assert_equal_with_printing(dedent("""
{
root_subject_types: (SubA,)
root_rules: (D, (Select(Exactly(B)), SelectDependencies(B, SubA, u'dependencies', field_types=(SubA, C,))), noop) of SubA
(B, (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),)
(D, (Select(Exactly(B)), SelectDependencies(B, SubA, u'dependencies', field_types=(SubA, C,))), noop) of SubA => ((B, (Select(SubA),), noop) of SubA, SubjectIsProduct(SubA), (B, (Select(SubA),), noop) of SubA,)
}""").strip(),
subgraph)
def test_successful_when_one_field_type_is_unfulfillable(self):
# NB We may want this to be a warning, since it may not be intentional
rules = [
TaskRule(B, [Select(SubA)], noop),
TaskRule(D, [Select(Exactly(B)), SelectDependencies(B, SubA, field_types=(SubA, C))], noop)
]
subgraph = self.create_subgraph(D, rules, SubA())
self.assert_equal_with_printing(dedent("""
digraph {
// root subject types: SubA
// root entries
"Select(D) for SubA" [color=blue]
"Select(D) for SubA" -> {"(D, (Select(B), SelectDependencies(B, SubA, field_types=(SubA, C,))), noop) of SubA"}
// internal entries
"(B, (Select(SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)"}
"(D, (Select(B), SelectDependencies(B, SubA, field_types=(SubA, C,))), noop) of SubA" -> {"(B, (Select(SubA),), noop) of SubA" "SubjectIsProduct(SubA)"}
}""").strip(),
subgraph)
def test_select_dependencies_multiple_field_types_all_resolvable(self):
rules = [
(Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop),
(B, (Select(Exactly(C, D)),), noop),
]
subgraph = self.create_subgraph(A, rules, SubA())
self.assert_equal_with_printing(dedent("""
digraph {
// root subject types: SubA
// root entries
"Select(A) for SubA" [color=blue]
"Select(A) for SubA" -> {"(A, (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA"}
// internal entries
"(A, (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA" -> {"SubjectIsProduct(SubA)" "(B, (Select(Exactly(C, D)),), noop) of C" "(B, (Select(Exactly(C, D)),), noop) of D"}
"(B, (Select(Exactly(C, D)),), noop) of C" -> {"SubjectIsProduct(C)"}
"(B, (Select(Exactly(C, D)),), noop) of D" -> {"SubjectIsProduct(D)"}
}""").strip(),
subgraph)
def test_select_dependencies_with_matching_intrinsic(self):
rules = [
(Exactly(A), (SelectDependencies(B, SubA, field_types=(C,)),), noop),
]
intrinsics = [
(B, C, noop),
]
graphmaker = GraphMaker(NodeBuilder.create(rules, intrinsics),
root_subject_fns=_suba_root_subject_fns)
subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A)
self.assert_equal_with_printing(dedent("""
{
root_subject_types: (SubA,)
root_rules: (Exactly(A), (SelectDependencies(B, SubA, u'dependencies', field_types=(C,)),), noop) of SubA
(Exactly(A), (SelectDependencies(B, SubA, u'dependencies', field_types=(C,)),), noop) of SubA => (SubjectIsProduct(SubA), IntrinsicRule(noop) of C,)
IntrinsicRule(noop) of C => (,)
}""").strip(),
subgraph)
def test_ruleset_unreachable_due_to_product_of_select_dependencies(self):
rules = [
(A, (SelectDependencies(B, SubA, field_types=(D,)),), noop),
]
intrinsics = [
(B, C, noop),
]
validator = RulesetValidator(NodeBuilder.create(rules, intrinsics),
goal_to_product={},
root_subject_fns={k: lambda p: Select(p) for k in (A,)})
with self.assertRaises(ValueError) as cm:
validator.validate()
self.assert_equal_with_printing(dedent("""
Rules with errors: 1
(A, (SelectDependencies(B, SubA, u'dependencies', field_types=(D,)),), noop):
Unreachable with subject types: Any
""").strip(),
str(cm.exception))
def test_descendant_specs(self):
"""Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory."""
spec = self.spec_parser.parse_spec('3rdparty/jvm::')
build_request = self.request_specs(['list'], spec)
walk = self.build_and_walk(build_request)
# Validate the root.
root, root_state = walk[0]
root_value = root_state.value
self.assertEqual(
DependenciesNode(
spec, None,
SelectDependencies(Address, Addresses,
field_types=(Address, ))), root)
self.assertEqual(list, type(root_value))
# Confirm that a few expected addresses are in the list.
self.assertIn(self.guava, root_value)
self.assertIn(self.managed_guava, root_value)
self.assertIn(self.managed_resolve_latest, root_value)
def test_sibling_specs(self):
"""Test that sibling Addresses are parsed in the 3rdparty/jvm directory."""
spec = self.spec_parser.parse_spec('3rdparty/jvm:')
build_request = self.request_specs(['list'], spec)
walk = self.build_and_walk(build_request)
# Validate the root.
root, root_state = walk[0]
root_value = root_state.value
self.assertEqual(
DependenciesNode(
spec, None,
SelectDependencies(Address, Addresses,
field_types=(Address, ))), root)
self.assertEqual(list, type(root_value))
# Confirm that an expected address is in the list.
self.assertIn(self.guava, root_value)
# And that an subdirectory address is not.
self.assertNotIn(self.managed_guava, root_value)
def test_select_dependencies_with_separate_types_for_subselectors(self):
rules = [(Exactly(A), (SelectDependencies(B, C,
field_types=(D, )), ), noop),
(B, (Select(D), ), noop), (C, (Select(SubA), ), noop)]
graphmaker = GraphMaker(RuleIndex.create(rules, tuple()),
root_subject_fns=_suba_root_subject_fns)
subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A)
self.assert_equal_with_printing(
dedent("""
{
root_subject_types: (SubA,)
root_rules:
Select(A) for SubA => ((Exactly(A), (SelectDependencies(B, C, field_types=(D,)),), noop) of SubA,)
all_rules:
(B, (Select(D),), noop) of D => (SubjectIsProduct(D),)
(C, (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),)
(Exactly(A), (SelectDependencies(B, C, field_types=(D,)),), noop) of SubA => ((C, (Select(SubA),), noop) of SubA, (B, (Select(D),), noop) of D,)
}""").strip(), subgraph)
def test_not_fulfillable_duplicated_dependency(self):
# If a rule depends on another rule+subject in two ways, and one of them is unfulfillable
# Only the unfulfillable one should be in the errors.
rules = _suba_root_rules + [
TaskRule(B, [Select(D)], noop),
TaskRule(D, [Select(A), SelectDependencies(A, SubA, field_types=(C,))], noop),
TaskRule(A, [Select(SubA)], noop)
]
validator = self.create_validator({}, rules)
with self.assertRaises(ValueError) as cm:
validator.assert_ruleset_valid()
self.assert_equal_with_printing(dedent("""
Rules with errors: 2
(B, (Select(D),), noop):
depends on unfulfillable (D, (Select(A), SelectDependencies(A, SubA, field_types=(C,))), noop) of SubA with subject types: SubA
(D, (Select(A), SelectDependencies(A, SubA, field_types=(C,))), noop):
depends on unfulfillable (A, (Select(SubA),), noop) of C with subject types: SubA
""").strip(),
str(cm.exception))
def test_select_dependencies_multiple_field_types_all_resolvable_with_deps(self):
rules = [
(Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop),
# for the C type, it'll just be a literal, but for D, it'll traverse one more edge
(B, (Select(C),), noop),
(C, (Select(D),), noop),
]
graphmaker = GraphMaker(NodeBuilder.create(rules, tuple()),
root_subject_fns=_suba_root_subject_fns)
subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A)
self.assert_equal_with_printing(dedent("""
{
root_subject_types: (SubA,)
root_rules: (Exactly(A), (SelectDependencies(B, SubA, u'dependencies', field_types=(C, D,)),), noop) of SubA
(B, (Select(C),), noop) of C => (SubjectIsProduct(C),)
(B, (Select(C),), noop) of D => ((C, (Select(D),), noop) of D,)
(C, (Select(D),), noop) of D => (SubjectIsProduct(D),)
(Exactly(A), (SelectDependencies(B, SubA, u'dependencies', field_types=(C, D,)),), noop) of SubA => (SubjectIsProduct(SubA), (B, (Select(C),), noop) of C, (B, (Select(C),), noop) of D,)
}""").strip(),
subgraph)
def test_not_fulfillable_duplicated_dependency(self):
# If a rule depends on another rule+subject in two ways, and one of them is unfulfillable
# Only the unfulfillable one should be in the errors.
rules = [(B, (Select(D), ), noop),
(D, (Select(A), SelectDependencies(A, SubA,
field_types=(C, ))), noop),
(A, (Select(SubA), ), noop)]
validator = RulesetValidator(RuleIndex.create(rules, tuple()),
goal_to_product={},
root_subject_fns=_suba_root_subject_fns)
with self.assertRaises(ValueError) as cm:
validator.validate()
self.assert_equal_with_printing(
dedent("""
Rules with errors: 2
(B, (Select(D),), noop):
depends on unfulfillable (D, (Select(A), SelectDependencies(A, SubA, field_types=(C,))), noop) of SubA with subject types: SubA
(D, (Select(A), SelectDependencies(A, SubA, field_types=(C,))), noop):
depends on unfulfillable (A, (Select(SubA),), noop) of C with subject types: SubA"""
).strip(), str(cm.exception))
def test_select_dependencies_with_matching_intrinsic(self):
rules = [
(Exactly(A), (SelectDependencies(B, SubA, field_types=(C,)),), noop),
]
intrinsics = [
(B, C, noop),
]
subgraph = self.create_subgraph_with_intrinsics(intrinsics, A, rules, SubA(),
_suba_root_subject_types)
#TODO perhaps intrinsics should be marked in the dot format somehow
self.assert_equal_with_printing(dedent("""
digraph {
// root subject types: SubA
// root entries
"Select(A) for SubA" [color=blue]
"Select(A) for SubA" -> {"(A, (SelectDependencies(B, SubA, field_types=(C,)),), noop) of SubA"}
// internal entries
"(A, (SelectDependencies(B, SubA, field_types=(C,)),), noop) of SubA" -> {"SubjectIsProduct(SubA)" "(B, (Select(C),), noop) of C"}
"(B, (Select(C),), noop) of C" -> {"SubjectIsProduct(C)"}
}""").strip(),
subgraph)
def test_select_dependencies_multiple_field_types_all_resolvable_with_deps(self):
rules = [
TaskRule(Exactly(A), [SelectDependencies(B, SubA, field_types=(C, D,))], noop),
# for the C type, it'll just be a literal, but for D, it'll traverse one more edge
TaskRule(B, [Select(C)], noop),
TaskRule(C, [Select(D)], noop),
]
subgraph = self.create_subgraph(A, rules, SubA())
self.assert_equal_with_printing(dedent("""
digraph {
// root subject types: SubA
// root entries
"Select(A) for SubA" [color=blue]
"Select(A) for SubA" -> {"(A, (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA"}
// internal entries
"(A, (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA" -> {"SubjectIsProduct(SubA)" "(B, (Select(C),), noop) of C" "(B, (Select(C),), noop) of D"}
"(B, (Select(C),), noop) of C" -> {"SubjectIsProduct(C)"}
"(B, (Select(C),), noop) of D" -> {"(C, (Select(D),), noop) of D"}
"(C, (Select(D),), noop) of D" -> {"SubjectIsProduct(D)"}
}""").strip(),
subgraph)
def create_fs_tasks():
"""Creates tasks that consume the native filesystem Node type."""
return [
# Glob execution.
(Stats, [
SelectProjection(Stats, Dir, ('directory', ), PathWildcard),
Select(PathWildcard)
], apply_path_wildcard),
(PathGlobs, [
SelectProjection(Dirs, Path, ('directory', ), PathLiteral),
Select(PathLiteral)
], apply_path_literal),
(PathGlobs, [
SelectProjection(Dirs, Dir, ('directory', ), PathDirWildcard),
Select(PathDirWildcard)
], apply_path_dir_wildcard),
] + [
# Link resolution.
(Dirs,
[Select(Stats),
SelectProjection(Dirs, Links,
('links', ), Stats)], resolve_dir_links),
(Dirs, [SelectProjection(Dirs, Path, ('path', ), ReadLink)], identity),
(Files,
[Select(Stats),
SelectProjection(Files, Links,
('links', ), Stats)], resolve_file_links),
(Files, [SelectProjection(Files, Path,
('path', ), ReadLink)], identity),
] + [
# TODO: These are boilerplatey: aggregation should become native:
# see https://github.com/pantsbuild/pants/issues/3169
(Stats, [SelectDependencies(Stats, PathGlobs)], merge_stats),
(Stats, [SelectDependencies(Stats, DirectoryListing, field='paths')
], merge_stats),
(Files, [SelectDependencies(Files, Links)], merge_files),
(Dirs, [SelectDependencies(Dirs, Links)], merge_dirs),
(FilesContent, [SelectDependencies(FileContent, Files)], FilesContent),
(FilesDigest, [SelectDependencies(FileDigest, Files)], FilesDigest),
]
def setup_json_scheduler(build_root, native):
"""Return a build graph and scheduler configured for BLD.json files under the given build root.
:rtype :class:`pants.engine.scheduler.LocalScheduler`
"""
symbol_table_cls = ExampleTable
# Register "literal" subjects required for these tasks.
# TODO: Replace with `Subsystems`.
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
build_patterns=('BLD.json',),
parser_cls=JsonParser)
source_roots = SourceRoots(('src/java','src/scala'))
scrooge_tool_address = Address.parse('src/scala/scrooge')
goals = {
'compile': Classpath,
# TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product.
'resolve': Classpath,
'list': Address,
GenGoal.name(): GenGoal,
'ls': Files,
'cat': FilesContent,
}
tasks = [
# Codegen
GenGoal.signature(),
(JavaSources,
[Select(ThriftSources),
SelectVariant(ApacheThriftJavaConfiguration, 'thrift')],
gen_apache_thrift),
(PythonSources,
[Select(ThriftSources),
SelectVariant(ApacheThriftPythonConfiguration, 'thrift')],
gen_apache_thrift),
(ScalaSources,
[Select(ThriftSources),
SelectVariant(ScroogeScalaConfiguration, 'thrift'),
SelectLiteral(scrooge_tool_address, Classpath)],
gen_scrooge_thrift),
(JavaSources,
[Select(ThriftSources),
SelectVariant(ScroogeJavaConfiguration, 'thrift'),
SelectLiteral(scrooge_tool_address, Classpath)],
gen_scrooge_thrift),
] + [
# scala dependency inference
(ScalaSources,
[Select(ScalaInferredDepsSources),
SelectDependencies(Address, ImportedJVMPackages, field_types=(JVMPackageName,))],
reify_scala_sources),
(ImportedJVMPackages,
[SelectProjection(FilesContent, PathGlobs, ('path_globs',), ScalaInferredDepsSources)],
extract_scala_imports),
(Address,
[Select(JVMPackageName),
SelectDependencies(AddressFamily, Dirs, field='stats', field_types=(Dir,))],
select_package_address),
(PathGlobs,
[Select(JVMPackageName),
SelectLiteral(source_roots, SourceRoots)],
calculate_package_search_path),
] + [
# Remote dependency resolution
(Classpath,
[Select(Jar)],
ivy_resolve),
(Jar,
[Select(ManagedJar),
SelectVariant(ManagedResolve, 'resolve')],
select_rev),
] + [
# Compilers
(Classpath,
[Select(ResourceSources)],
isolate_resources),
(Classpath,
[Select(BuildPropertiesConfiguration)],
write_name_file),
# NB: Not sure these SelectDependencies should allow Jar, but they currently produce jars.
(Classpath,
[Select(JavaSources),
SelectDependencies(Classpath, JavaSources, field_types=(Address, Jar))],
javac),
(Classpath,
[Select(ScalaSources),
SelectDependencies(Classpath, ScalaSources, field_types=(Address, Jar))],
scalac),
] + (
create_graph_tasks(address_mapper, symbol_table_cls)
) + (
create_fs_tasks()
)
project_tree = FileSystemProjectTree(build_root)
return LocalScheduler(goals,
tasks,
project_tree,
native,
graph_lock=None)
raise ResolvedTypeMismatchError(e)
# Let factories replace the hydrated object.
if isinstance(item, SerializableFactory):
item = item.create()
# Finally make sure objects that can self-validate get a chance to do so.
if isinstance(item, Validatable):
item.validate()
return item
@rule(BuildFileAddresses,
[Select(AddressMapper),
SelectDependencies(AddressFamily, BuildDirs, field_types=(Dir,)),
Select(_SPECS_CONSTRAINT)])
def addresses_from_address_families(address_mapper, address_families, spec):
"""Given a list of AddressFamilies and a Spec, return matching Addresses.
Raises a ResolveError if:
- there were no matching AddressFamilies, or
- the Spec matches no addresses for SingleAddresses.
"""
if not address_families:
raise ResolveError('Path "{}" contains no BUILD files.'.format(spec.directory))
def exclude_address(address):
if address_mapper.exclude_patterns:
address_str = address.spec
return any(p.search(address_str) is not None for p in address_mapper.exclude_patterns)
extensions = ('.scala',)
class JVMPackageName(datatype('JVMPackageName', ['name'])):
"""A typedef to represent a fully qualified JVM package name."""
pass
class SourceRoots(datatype('SourceRoots', ['srcroots'])):
"""Placeholder for the SourceRoot subsystem."""
@printing_func
@rule(Address,
[Select(JVMPackageName),
SelectDependencies(AddressFamily, Snapshot, field='dir_stats', field_types=(Dir,))])
def select_package_address(jvm_package_name, address_families):
"""Return the Address from the given AddressFamilies which provides the given package."""
addresses = [address for address_family in address_families
for address in address_family.addressables.keys()]
if len(addresses) == 0:
raise ValueError('No targets existed in {} to provide {}'.format(
address_families, jvm_package_name))
elif len(addresses) > 1:
raise ValueError('Multiple targets might be able to provide {}:\n {}'.format(
jvm_package_name, '\n '.join(str(a) for a in addresses)))
return addresses[0].to_address()
@printing_func
@rule(PathGlobs, [Select(JVMPackageName), Select(SourceRoots)])
@rule(HydratedField, [
Select(SourcesField),
SelectProjection(Snapshot, PathGlobs, 'path_globs', SourcesField)
])
def hydrate_sources(sources_field, snapshot):
"""Given a SourcesField and a Snapshot for its path_globs, create an EagerFilesetWithSpec."""
fileset_with_spec = _eager_fileset_with_spec(
sources_field.address.spec_path, sources_field.filespecs, snapshot)
return HydratedField(sources_field.arg, fileset_with_spec)
@rule(HydratedField, [
Select(BundlesField),
SelectDependencies(
Snapshot, BundlesField, 'path_globs_list', field_types=(PathGlobs, ))
])
def hydrate_bundles(bundles_field, snapshot_list):
"""Given a BundlesField and a Snapshot for each of its filesets create a list of BundleAdaptors."""
bundles = []
zipped = zip(bundles_field.bundles, bundles_field.filespecs_list,
snapshot_list)
for bundle, filespecs, snapshot in zipped:
spec_path = bundles_field.address.spec_path
kwargs = bundle.kwargs()
kwargs['fileset'] = _eager_fileset_with_spec(
getattr(bundle, 'rel_path', spec_path), filespecs, snapshot)
bundles.append(BundleAdaptor(**kwargs))
return HydratedField('bundles', bundles)
@rule(TransitiveHydratedTargets, [
SelectTransitive(HydratedTarget,
BuildFileAddresses,
field_types=(Address, ),
field='addresses')
])
def transitive_hydrated_targets(targets):
"""Recursively requests HydratedTarget instances, which will result in an eager, transitive graph walk."""
return TransitiveHydratedTargets(targets)
@rule(HydratedTargets, [
SelectDependencies(HydratedTarget,
BuildFileAddresses,
field_types=(Address, ),
field='addresses')
])
def hydrated_targets(targets):
"""Requests HydratedTarget instances."""
return HydratedTargets(targets)
class HydratedField(datatype('HydratedField', ['name', 'value'])):
"""A wrapper for a fully constructed replacement kwarg for a HydratedTarget."""
def hydrate_target(target_adaptor, hydrated_fields):
"""Construct a HydratedTarget from a TargetAdaptor and hydrated versions of its adapted fields."""
# Hydrate the fields of the adaptor and re-construct it.
kwargs = target_adaptor.kwargs()
def step(self, step_context):
waiting_nodes = []
# Get the binary.
binary_state = step_context.select_for(Select(self.snapshotted_process.binary_type),
subject=self.subject,
variants=self.variants)
if type(binary_state) is Throw:
return binary_state
elif type(binary_state) is Waiting:
waiting_nodes.extend(binary_state.dependencies)
elif type(binary_state) is Noop:
return Noop("Couldn't find binary: {}".format(binary_state))
elif type(binary_state) is not Return:
State.raise_unrecognized(binary_state)
# Create the request from the request callback after resolving its input clauses.
input_values = []
for input_selector in self.snapshotted_process.input_selectors:
sn_state = step_context.select_for(input_selector, self.subject, self.variants)
if type(sn_state) is Waiting:
waiting_nodes.extend(sn_state.dependencies)
elif type(sn_state) is Return:
input_values.append(sn_state.value)
elif type(sn_state) is Noop:
if input_selector.optional:
input_values.append(None)
else:
return Noop('Was missing value for (at least) input {}'.format(input_selector))
elif type(sn_state) is Throw:
return sn_state
else:
State.raise_unrecognized(sn_state)
if waiting_nodes:
return Waiting(waiting_nodes)
# Now that we've returned on waiting, we can assume that relevant inputs have values.
try:
process_request = self.snapshotted_process.input_conversion(*input_values)
except Exception as e:
return Throw(e)
# Request snapshots for the snapshot_subjects from the process request.
snapshot_subjects_value = []
if process_request.snapshot_subjects:
snapshot_subjects_state = step_context.select_for(SelectDependencies(Snapshot,
SnapshottedProcessRequest,
'snapshot_subjects',
field_types=(Files,)),
process_request,
self.variants)
if type(snapshot_subjects_state) is not Return:
return snapshot_subjects_state
snapshot_subjects_value = snapshot_subjects_state.value
# Ready to run.
execution = _Process(step_context.snapshot_archive_root,
process_request,
binary_state.value,
snapshot_subjects_value,
self.snapshotted_process.output_conversion)
return Runnable(_execute, (execution,))
