def test_select_dependencies_with_separate_types_for_subselectors(self): rules = [ TaskRule(Exactly(A), [SelectDependencies(B, C, field_types=(D, ))], noop), TaskRule(B, [Select(D)], noop), TaskRule(C, [Select(SubA)], noop) ] subgraph = self.create_subgraph(A, rules, SubA()) self.assert_equal_with_printing( dedent(""" digraph { // root subject types: SubA // root entries "Select(A) for SubA" [color=blue] "Select(A) for SubA" -> {"(A, (SelectDependencies(B, C, field_types=(D,)),), noop) of SubA"} // internal entries "(A, (SelectDependencies(B, C, field_types=(D,)),), noop) of SubA" -> {"(C, (Select(SubA),), noop) of SubA" "(B, (Select(D),), noop) of D"} "(B, (Select(D),), noop) of D" -> {"SubjectIsProduct(D)"} "(C, (Select(SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)"} }""").strip(), subgraph)
def test_select_dependencies_multiple_field_types_all_resolvable_with_deps(self): rules = [ (Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop), # for the C type, it'll just be a literal, but for D, it'll traverse one more edge (B, (Select(C),), noop), (C, (Select(D),), noop), ] graphmaker = GraphMaker(NodeBuilder.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) root_rules: (Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA (B, (Select(C),), noop) of C => (SubjectIsProduct(C),) (B, (Select(C),), noop) of D => ((C, (Select(D),), noop) of D,) (C, (Select(D),), noop) of D => (SubjectIsProduct(D),) (Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA => (SubjectIsProduct(SubA), (B, (Select(C),), noop) of C, (B, (Select(C),), noop) of D,) }""").strip(), subgraph)
def resolve(self, spec): # TODO reenable using SelectDependencies select = Select(UnhydratedStructs) request = self.scheduler.selection_request([(select, spec)]) result = LocalSerialEngine(self.scheduler).execute(request) if result.error: raise result.error # Expect a single root. state, = result.root_products.values() if type(state) is Throw: raise Exception(state.exc) return state.value.dependencies
def test_not_fulfillable_duplicated_dependency(self): # If a rule depends on another rule+subject in two ways, and one of them is unfulfillable # Only the unfulfillable one should be in the errors. rules = [ (B, (Select(D),), noop), (D, (Select(A), SelectDependencies(A, SubA, field_types=(C,))), noop), (A, (Select(SubA),), noop) ] validator = RulesetValidator(NodeBuilder.create(rules, tuple()), goal_to_product={}, root_subject_fns=_suba_root_subject_fns) with self.assertRaises(ValueError) as cm: validator.validate() self.assert_equal_with_printing(dedent(""" Rules with errors: 2 (B, (Select(D),), noop): depends on unfulfillable (D, (Select(A), SelectDependencies(A, SubA, field_types=(C,))), noop) of SubA with subject types: SubA (D, (Select(A), SelectDependencies(A, SubA, field_types=(C,))), noop): depends on unfulfillable (A, (Select(SubA),), noop) of C with subject types: SubA""").strip(), str(cm.exception))
def __init__(self, goals, tasks, project_tree, native, graph_lock=None): """ :param goals: A dict from a goal name to a product type. A goal is just an alias for a particular (possibly synthetic) product. :param tasks: A set of (output, input selection clause, task function) triples which is used to compute values in the product graph. :param project_tree: An instance of ProjectTree for the current build root. :param native: An instance of engine.subsystem.native.Native. :param graph_lock: A re-entrant lock to use for guarding access to the internal product Graph instance. Defaults to creating a new threading.RLock(). """ self._products_by_goal = goals self._project_tree = project_tree self._native = native self._product_graph_lock = graph_lock or threading.RLock() self._run_count = 0 # TODO: The only (?) case where we use inheritance rather than exact type unions. has_products_constraint = SubclassesOf(HasProducts) # Create the ExternContext, and the native Scheduler. self._scheduler = native.new_scheduler(has_products_constraint, constraint_for(Address), constraint_for(Variants)) self._execution_request = None # Validate and register all provided and intrinsic tasks. # TODO: This bounding of input Subject types allows for closed-world validation, but is not # strictly necessary for execution. We might eventually be able to remove it by only executing # validation below the execution roots (and thus not considering paths that aren't in use). select_product = lambda product: Select(product) root_selector_fns = { Address: select_product, AscendantAddresses: select_product, DescendantAddresses: select_product, PathGlobs: select_product, SiblingAddresses: select_product, SingleAddress: select_product, } intrinsics = create_fs_intrinsics(project_tree) + create_snapshot_intrinsics(project_tree) singletons = create_snapshot_singletons(project_tree) rule_index = RuleIndex.create(tasks, intrinsics, singletons) RulesetValidator(rule_index, goals, root_selector_fns).validate() self._register_tasks(rule_index.tasks) self._register_intrinsics(rule_index.intrinsics) self._register_singletons(rule_index.singletons)
def test_ruleset_with_missing_product_type(self): rules = [TaskRule(A, [Select(B)], noop)] root_subject_types = {SubA} scheduler = self.create_native_scheduler(root_subject_types, rules) with self.assertRaises(ValueError) as cm: scheduler.assert_ruleset_valid() self.assert_equal_with_printing( dedent(""" Rules with errors: 1 (A, (Select(B),), noop): no matches for Select(B) with subject types: SubA """).strip(), str(cm.exception))
def test_smallest_full_test(self): rules = [TaskRule(Exactly(A), [Select(SubA)], noop)] fullgraph = self.create_full_graph({SubA}, RuleIndex.create(rules)) self.assert_equal_with_printing( dedent(""" digraph { // root subject types: SubA // root entries "Select(A) for SubA" [color=blue] "Select(A) for SubA" -> {"(A, (Select(SubA),), noop) of SubA"} // internal entries "(A, (Select(SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)"} }""").strip(), fullgraph)
def step(self, step_context): selector = Select(Files) node = step_context.select_node(selector, self.subject, self.variants) select_state = step_context.get(node) if type(select_state) in {Waiting, Noop, Throw}: return select_state elif type(select_state) is not Return: State.raise_unrecognized(select_state) file_list = select_state.value snapshot = _create_snapshot_archive(file_list, step_context) return Return(snapshot)
def test_noop_removal_transitive(self): # If a noop-able rule has rules that depend on it, # they should be removed from the graph. rules = [ (Exactly(B), (Select(C),), noop), (Exactly(A), (Select(B),), noop), (Exactly(A), tuple(), noop), ] intrinsics = [ (D, C, BoringRule(C)) ] graphmaker = GraphMaker(NodeBuilder.create(rules, intrinsics), root_subject_fns=_suba_root_subject_fns, ) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) root_rules: (Exactly(A), (), noop) of SubA (Exactly(A), (), noop) of SubA => (,) }""").strip(), subgraph)
def test_javac_compilation_example(self): sources = PathGlobs.create('', files=['scheduler_inputs/src/java/simple/Simple.java']) scheduler = self.mk_scheduler_in_example_fs([ SnapshottedProcess.create(ClasspathEntry, Javac, (Select(Files), Select(Snapshot), SelectLiteral(JavaOutputDir('build'), JavaOutputDir)), java_sources_to_javac_args, process_result_to_classpath_entry), [Javac, [], Javac] ]) request = scheduler.execution_request( [ClasspathEntry], [sources]) LocalSerialEngine(scheduler).reduce(request) root_entries = scheduler.root_entries(request).items() self.assertEquals(1, len(root_entries)) state = self.assertFirstEntryIsReturn(root_entries, scheduler) classpath_entry = state.value self.assertIsInstance(classpath_entry, ClasspathEntry) self.assertTrue(os.path.exists(os.path.join(classpath_entry.path, 'simple', 'Simple.class')))
def test_select_dependencies_multiple_field_types_all_resolvable_with_deps(self): rules = [ TaskRule(Exactly(A), [SelectDependencies(B, SubA, field_types=(C, D,))], noop), # for the C type, it'll just be a literal, but for D, it'll traverse one more edge TaskRule(B, [Select(C)], noop), TaskRule(C, [Select(D)], noop), ] subgraph = self.create_subgraph(A, rules, SubA()) self.assert_equal_with_printing(dedent(""" digraph { // root subject types: SubA // root entries "Select(A) for SubA" [color=blue] "Select(A) for SubA" -> {"(A, (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA"} // internal entries "(A, (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA" -> {"SubjectIsProduct(SubA)" "(B, (Select(C),), noop) of C" "(B, (Select(C),), noop) of D"} "(B, (Select(C),), noop) of C" -> {"SubjectIsProduct(C)"} "(B, (Select(C),), noop) of D" -> {"(C, (Select(D),), noop) of D"} "(C, (Select(D),), noop) of D" -> {"SubjectIsProduct(D)"} }""").strip(), subgraph)
def test_root_tuple_removed_when_no_matches(self): rules = [ RootRule(C), RootRule(D), TaskRule(Exactly(A), [Select(C)], noop), TaskRule(Exactly(B), [Select(D), Select(A)], noop), ] fullgraph = self.create_full_graph(rules, validate=False) self.assert_equal_with_printing( dedent(""" digraph { // root subject types: C, D // root entries "Select(A) for C" [color=blue] "Select(A) for C" -> {"(A, [Select(C)], noop) for C"} "Select(B) for (C+D)" [color=blue] "Select(B) for (C+D)" -> {"(B, [Select(D), Select(A)], noop) for (C+D)"} // internal entries "(A, [Select(C)], noop) for C" -> {"Param(C)"} "(B, [Select(D), Select(A)], noop) for (C+D)" -> {"(A, [Select(C)], noop) for C" "Param(D)"} }""").strip(), fullgraph)
def create_snapshot_tasks(project_tree): """TODO: Delete these. Necessary because the intrinsic will not trigger conversions to `Files` on a PathGlobs object. Instead, should replace the intrinsic with an uncacheable task, or have it depend on an uncacheable singleton. """ def ptree(func): return functools.partial(func, project_tree, snapshot_directory(project_tree)) return [ (Snapshot, [Select(Files)], ptree(create_snapshot_archive)), ]
def test_not_fulfillable_duplicated_dependency(self): # If a rule depends on another rule+subject in two ways, and one of them is unfulfillable # Only the unfulfillable one should be in the errors. rules = _suba_root_rules + [ TaskRule(B, [Select(D)], noop), TaskRule( D, [Select(A), SelectDependencies(A, SubA, field_types=(C, ))], noop), TaskRule(A, [Select(SubA)], noop) ] validator = self.create_validator({}, rules) with self.assertRaises(ValueError) as cm: validator.assert_ruleset_valid() self.assert_equal_with_printing( dedent(""" Rules with errors: 2 (B, (Select(D),), noop): depends on unfulfillable (D, (Select(A), SelectDependencies(A, SubA, field_types=(C,))), noop) of SubA with subject types: SubA (D, (Select(A), SelectDependencies(A, SubA, field_types=(C,))), noop): depends on unfulfillable (A, (Select(SubA),), noop) of C with subject types: SubA """).strip(), str(cm.exception))
def test_integration_concat_with_snapshot_subjects_test(self): scheduler = self.mk_scheduler_in_example_fs([ # subject to files / product of subject to files for snapshot. SnapshottedProcess.create( product_type=Concatted, binary_type=ShellCatToOutFile, input_selectors=(Select(Files), Select(Snapshot)), input_conversion= file_list_to_args_for_cat_with_snapshot_subjects_and_output_file, output_conversion=process_result_to_concatted_from_outfile), [ShellCatToOutFile, [], ShellCatToOutFile], ]) request = scheduler.execution_request( [Concatted], [PathGlobs.create('', globs=['fs_test/a/b/*'])]) LocalSerialEngine(scheduler).reduce(request) root_entries = scheduler.root_entries(request).items() self.assertEquals(1, len(root_entries)) state = self.assertFirstEntryIsReturn(root_entries, scheduler) concatted = state.value self.assertEqual(Concatted('one\ntwo\n'), concatted)
def test_ruleset_with_missing_product_type(self): rules = [(A, (Select(B), ), noop)] validator = RulesetValidator(NodeBuilder.create(rules), goal_to_product=dict(), root_subject_types=tuple()) with self.assertRaises(ValueError) as cm: validator.validate() self.assertEquals( dedent(""" Found 1 rules with errors: (A, (Select(B),), noop) There is no producer of Select(B) """).strip(), str(cm.exception))
def test_successful_when_one_field_type_is_unfulfillable(self): # NB We may want this to be a warning, since it may not be intentional rules = [ TaskRule(B, [Select(SubA)], noop), TaskRule(D, [ Select(Exactly(B)), SelectDependencies(B, SubA, field_types=(SubA, C)) ], noop) ] subgraph = self.create_subgraph(D, rules, SubA()) self.assert_equal_with_printing( dedent(""" digraph { // root subject types: SubA // root entries "Select(D) for SubA" [color=blue] "Select(D) for SubA" -> {"(D, (Select(B), SelectDependencies(B, SubA, field_types=(SubA, C,))), noop) of SubA"} // internal entries "(B, (Select(SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)"} "(D, (Select(B), SelectDependencies(B, SubA, field_types=(SubA, C,))), noop) of SubA" -> {"(B, (Select(SubA),), noop) of SubA" "SubjectIsProduct(SubA)"} }""").strip(), subgraph)
def create_legacy_graph_tasks(): """Create tasks to recursively parse the legacy graph.""" return [ # Recursively requests the dependencies and adapted fields of TargetAdaptors, which # will result in an eager, transitive graph walk. (LegacyTarget, [ Select(TargetAdaptor), SelectDependencies(LegacyTarget, TargetAdaptor, 'dependencies'), SelectDependencies(HydratedField, TargetAdaptor, 'field_adaptors') ], reify_legacy_graph), (HydratedField, [ Select(SourcesField), SelectProjection(FilesContent, PathGlobs, ('path_globs', ), SourcesField), SelectProjection(Files, PathGlobs, ('excluded_path_globs', ), SourcesField) ], hydrate_sources), (HydratedField, [ Select(BundlesField), SelectDependencies(FilesContent, BundlesField, 'path_globs_list'), SelectDependencies(Files, BundlesField, 'excluded_path_globs_list') ], hydrate_bundles), ]
def test_javac_compilation_example(self): sources = PathGlobs.create( '', include=['scheduler_inputs/src/java/simple/Simple.java']) scheduler = self.mk_scheduler_in_example_fs([ SnapshottedProcess.create( ClasspathEntry, Javac, (Select(Snapshot), Select(JavaOutputDir)), java_sources_to_javac_args, process_result_to_classpath_entry), SingletonRule(JavaOutputDir, JavaOutputDir('build')), SingletonRule(Javac, Javac()), ]) request = scheduler.execution_request([ClasspathEntry], [sources]) root_entries = scheduler.execute(request).root_products self.assertEquals(1, len(root_entries)) state = self.assertFirstEntryIsReturn(root_entries, scheduler) classpath_entry = state.value self.assertIsInstance(classpath_entry, ClasspathEntry) self.assertTrue( os.path.exists( os.path.join(classpath_entry.path, 'simple', 'Simple.class')))
def create_graph_rules(address_mapper, symbol_table): """Creates tasks used to parse Structs from BUILD files. :param address_mapper_key: The subject key for an AddressMapper instance. :param symbol_table: A SymbolTable instance to provide symbols for Address lookups. """ symbol_table_constraint = symbol_table.constraint() return [ TaskRule(BuildFilesCollection, [SelectDependencies(BuildFiles, BuildDirs, field_types=(Dir,))], BuildFilesCollection), # A singleton to provide the AddressMapper. SingletonRule(AddressMapper, address_mapper), # Support for resolving Structs from Addresses. TaskRule( symbol_table_constraint, [Select(AddressMapper), Select(UnhydratedStruct), SelectDependencies(symbol_table_constraint, UnhydratedStruct, field_types=(Address,))], hydrate_struct ), resolve_unhydrated_struct, # BUILD file parsing. parse_address_family, build_files, buildfile_path_globs_for_dir, # Spec handling: locate directories that contain build files, and request # AddressFamilies for each of them. addresses_from_address_families, filter_build_dirs, spec_to_globs, # Root rules representing parameters that might be provided via root subjects. RootRule(Address), RootRule(BuildFileAddress), RootRule(BuildFileAddresses), RootRule(Specs), ]
def test_multiple_depend_on_same_rule(self): @rule(B, [Select(A)]) def b_from_a(a): pass @rule(C, [Select(A)]) def c_from_a(a): pass @rule(A, [Select(SubA)]) def a_from_suba(suba): pass rules = _suba_root_rules + [ b_from_a, c_from_a, a_from_suba, ] subgraph = self.create_full_graph(rules) self.assert_equal_with_printing( dedent(""" digraph { // root subject types: SubA // root entries "Select(A) for SubA" [color=blue] "Select(A) for SubA" -> {"(A, [Select(SubA)], a_from_suba) for SubA"} "Select(B) for SubA" [color=blue] "Select(B) for SubA" -> {"(B, [Select(A)], b_from_a) for SubA"} "Select(C) for SubA" [color=blue] "Select(C) for SubA" -> {"(C, [Select(A)], c_from_a) for SubA"} // internal entries "(A, [Select(SubA)], a_from_suba) for SubA" -> {"Param(SubA)"} "(B, [Select(A)], b_from_a) for SubA" -> {"(A, [Select(SubA)], a_from_suba) for SubA"} "(C, [Select(A)], c_from_a) for SubA" -> {"(A, [Select(SubA)], a_from_suba) for SubA"} }""").strip(), subgraph)
def test_gen(self): build_request = self.request(['gen'], self.thrift) root, = self.build(build_request) # Root: expect the synthetic GenGoal product. self.assert_root(root, self.thrift, GenGoal("non-empty input to satisfy the Goal constructor")) variants = {'thrift': 'apache_java'} # Expect ThriftSources to have been selected. self.assert_select_for_subjects(walk, Select(ThriftSources), [self.thrift], variants=variants) # Expect an ApacheThriftJavaConfiguration to have been used via the default Variants. self.assert_select_for_subjects(walk, SelectVariant(ApacheThriftJavaConfiguration, variant_key='thrift'), [self.thrift], variants=variants)
def test_no_include_trace_error_multiple_paths_raises_executionerror(self): rules = [ TaskRule(A, [Select(B)], nested_raise), ] engine = self.create_engine({B}, rules, include_trace_on_error=False) with self.assertRaises(Exception) as cm: list(engine.product_request(A, subjects=[B(), B()])) self.assert_equal_with_printing( dedent(''' Multiple exceptions encountered: Exception: An exception for B Exception: An exception for B''').lstrip(), str(cm.exception))
def test_descendant_specs(self): """Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec('3rdparty/jvm::') selector = Select(BuildFileAddresses) build_request = self.scheduler.selection_request([(selector, spec)]) ((subject, _), root), = self.build(build_request) # Validate the root. self.assertEqual(spec, subject) self.assertEqual(BuildFileAddresses, type(root.value)) # Confirm that a few expected addresses are in the list. self.assertIn(self.guava, root.value.dependencies) self.assertIn(self.managed_guava, root.value.dependencies) self.assertIn(self.managed_resolve_latest, root.value.dependencies)
def test_sibling_specs(self): """Test that sibling Addresses are parsed in the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec('3rdparty/jvm:') selector = Select(BuildFileAddresses) build_request = self.scheduler.selection_request([(selector, spec)]) ((subject, _), root), = self.build(build_request) # Validate the root. self.assertEqual(spec, subject) self.assertEqual(BuildFileAddresses, type(root.value)) # Confirm that an expected address is in the list. self.assertIn(self.guava, root.value.dependencies) # And that a subdirectory address is not. self.assertNotIn(self.managed_guava, root.value.dependencies)
def test_single_rule_depending_on_subject_selection(self): rules = [ (Exactly(A), (Select(SubA),), noop) ] graphmaker = GraphMaker(NodeBuilder.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) root_rules: (Exactly(A), (Select(SubA),), noop) of SubA (Exactly(A), (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) }""").strip(), subgraph)
def test_single_rule_depending_on_subject_selection(self): rules = [TaskRule(Exactly(A), [Select(SubA)], noop)] subgraph = self.create_subgraph(A, rules, SubA()) self.assert_equal_with_printing( dedent(""" digraph { // root subject types: SubA // root entries "Select(A) for SubA" [color=blue] "Select(A) for SubA" -> {"(A, [Select(SubA)], noop) for SubA"} // internal entries "(A, [Select(SubA)], noop) for SubA" -> {"Param(SubA)"} }""").strip(), subgraph)
def test_ruleset_with_missing_product_type(self): @rule(A, [Select(B)]) def a_from_b_noop(b): pass rules = _suba_root_rules + [a_from_b_noop] with self.assertRaises(Exception) as cm: create_scheduler(rules) self.assert_equal_with_printing( dedent(""" Rules with errors: 1 (A, [Select(B)], a_from_b_noop): No rule was available to compute B with parameter type SubA """).strip(), str(cm.exception))
def create_fs_tasks(): """Creates tasks that consume the native filesystem Node type.""" return [ # Glob execution. (Paths, [SelectDependencies(Paths, PathGlobs)], merge_paths), (Paths, [Select(PathRoot)], apply_path_root), (Paths, [ SelectProjection(DirectoryListing, Dir, ('canonical_stat', ), PathWildcard), Select(PathWildcard) ], apply_path_wildcard), (PathGlobs, [ SelectProjection(Dirs, Paths, ('paths', ), FilteredPaths), Select(PathDirWildcard) ], apply_path_dir_wildcard), (FilteredPaths, [ SelectProjection(DirectoryListing, Dir, ('canonical_stat', ), PathDirWildcard), Select(PathDirWildcard) ], filter_paths), ] + [ # Link resolution. (Dirs, [ Select(Paths), SelectDependencies(Dirs, Paths, field='link_stats') ], resolve_dir_links), (Files, [ Select(Paths), SelectDependencies(Files, Paths, field='link_stats') ], resolve_file_links), (Dirs, [SelectProjection(Dirs, PathGlobs, ('path_globs', ), ReadLink)], resolve_link), (Files, [SelectProjection(Files, PathGlobs, ('path_globs', ), ReadLink)], resolve_link), ] + [ # File content. (FilesContent, [ Select(Files), SelectDependencies(FileContent, Files, field='stats') ], files_content), (FilesDigest, [Select(Files), SelectDependencies(FileDigest, Files, field='stats')], files_digest), ]
def test_no_include_trace_error_multiple_paths_raises_executionerror(self): rules = [ RootRule(B), TaskRule(A, [Select(B)], nested_raise), ] scheduler = self.scheduler(rules, include_trace_on_error=False) with self.assertRaises(ExecutionError) as cm: list(scheduler.product_request(A, subjects=[B(), B()])) self.assert_equal_with_printing(dedent(''' 2 Exceptions encountered: Exception: An exception for B Exception: An exception for B''').lstrip(), str(cm.exception))