def execute_request(self, scheduler, product, *subjects):
   """Creates, runs, and returns an ExecutionRequest for the given product and subjects."""
   request = scheduler.execution_request([product], subjects)
   engine = LocalSerialEngine(scheduler)
   res = engine.execute(request)
   if res.error:
     raise res.error
   return request
Beispiel #2
0
 def execute_request(self, scheduler, product, *subjects):
     """Creates, runs, and returns an ExecutionRequest for the given product and subjects."""
     request = scheduler.execution_request([product], subjects)
     engine = LocalSerialEngine(scheduler)
     res = engine.execute(request)
     if res.error:
         raise res.error
     return request
Beispiel #3
0
class SchedulerTest(unittest.TestCase):

  _native = init_native()

  def setUp(self):
    build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs')
    self.spec_parser = CmdLineSpecParser(build_root)
    self.scheduler = setup_json_scheduler(build_root, self._native)
    self.engine = LocalSerialEngine(self.scheduler)

    self.guava = Address.parse('3rdparty/jvm:guava')
    self.thrift = Address.parse('src/thrift/codegen/simple')
    self.java = Address.parse('src/java/codegen/simple')
    self.java_simple = Address.parse('src/java/simple')
    self.java_multi = Address.parse('src/java/multiple_classpath_entries')
    self.no_variant_thrift = Address.parse('src/java/codegen/selector:conflict')
    self.unconfigured_thrift = Address.parse('src/thrift/codegen/unconfigured')
    self.resources = Address.parse('src/resources/simple')
    self.consumes_resources = Address.parse('src/java/consumes_resources')
    self.consumes_managed_thirdparty = Address.parse('src/java/managed_thirdparty')
    self.managed_guava = Address.parse('3rdparty/jvm/managed:guava')
    self.managed_hadoop = Address.parse('3rdparty/jvm/managed:hadoop-common')
    self.managed_resolve_latest = Address.parse('3rdparty/jvm/managed:latest-hadoop')
    self.inferred_deps = Address.parse('src/scala/inferred_deps')

  def assert_select_for_subjects(self, walk, selector, subjects, variants=None):
    raise ValueError(walk)

  def build(self, build_request):
    """Execute the given request and return roots as a list of ((subject, product), value) tuples."""
    result = self.engine.execute(build_request)
    self.assertIsNone(result.error)
    return self.scheduler.root_entries(build_request).items()

  def request(self, goals, *subjects):
    return self.scheduler.build_request(goals=goals, subjects=subjects)

  def assert_root(self, root, subject, return_value):
    """Asserts that the given root has the given result."""
    self.assertEquals(subject, root[0][0])
    self.assertEquals(Return(return_value), root[1])

  def assert_root_failed(self, root, subject, msg_str):
    """Asserts that the root was a Throw result containing the given msg string."""
    self.assertEquals(subject, root[0][0])
    self.assertEquals(Throw, type(root[1]))
    self.assertIn(msg_str, str(root[1].exc))

  def test_compile_only_3rdparty(self):
    build_request = self.request(['compile'], self.guava)
    root, = self.build(build_request)
    self.assert_root(root, self.guava, Classpath(creator='ivy_resolve'))

  @unittest.skip('Skipped to expedite landing #3821; see: #4027.')
  def test_compile_only_3rdparty_internal(self):
    build_request = self.request(['compile'], '3rdparty/jvm:guava')
    root, = self.build(build_request)

    # Expect a SelectNode for each of the Jar/Classpath.
    self.assert_select_for_subjects(walk, Select(Jar), [self.guava])
    self.assert_select_for_subjects(walk, Select(Classpath), [self.guava])

  @unittest.skip('Skipped to expedite landing #3821; see: #4020.')
  def test_gen(self):
    build_request = self.request(['gen'], self.thrift)
    root, = self.build(build_request)

    # Root: expect the synthetic GenGoal product.
    self.assert_root(root, self.thrift, GenGoal("non-empty input to satisfy the Goal constructor"))

    variants = {'thrift': 'apache_java'}
    # Expect ThriftSources to have been selected.
    self.assert_select_for_subjects(walk, Select(ThriftSources), [self.thrift], variants=variants)
    # Expect an ApacheThriftJavaConfiguration to have been used via the default Variants.
    self.assert_select_for_subjects(walk, SelectVariant(ApacheThriftJavaConfiguration,
                                                        variant_key='thrift'),
                                    [self.thrift],
                                    variants=variants)

  @unittest.skip('Skipped to expedite landing #3821; see: #4020.')
  def test_codegen_simple(self):
    build_request = self.request(['compile'], self.java)
    root, = self.build(build_request)

    # The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants.
    subjects = [self.guava, self.java, self.thrift]
    variant_subjects = [
        Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2', type_alias='jar'),
        Jar(org='commons-lang', name='commons-lang', rev='2.5', type_alias='jar'),
        Address.parse('src/thrift:slf4j-api')]

    # Root: expect a DependenciesNode depending on a SelectNode with compilation via javac.
    self.assert_root(root, self.java, Classpath(creator='javac'))

    # Confirm that exactly the expected subjects got Classpaths.
    self.assert_select_for_subjects(walk, Select(Classpath), subjects)
    self.assert_select_for_subjects(walk, Select(Classpath), variant_subjects,
                                    variants={'thrift': 'apache_java'})

  def test_consumes_resources(self):
    build_request = self.request(['compile'], self.consumes_resources)
    root, = self.build(build_request)
    self.assert_root(root, self.consumes_resources, Classpath(creator='javac'))

  @unittest.skip('Skipped to expedite landing #3821; see: #4027.')
  def test_consumes_resources_internal(self):
    build_request = self.request(['compile'], self.consumes_resources)
    root, = self.build(build_request)

    # Confirm a classpath for the resources target and other subjects. We know that they are
    # reachable from the root (since it was involved in this walk).
    subjects = [self.resources,
                self.consumes_resources,
                self.guava]
    self.assert_select_for_subjects(walk, Select(Classpath), subjects)

  @unittest.skip('Skipped to expedite landing #3821; see: #4020.')
  def test_managed_resolve(self):
    """A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars."""
    build_request = self.request(['compile'], self.consumes_managed_thirdparty)
    root, = self.build(build_request)

    # Validate the root.
    self.assert_root(root, self.consumes_managed_thirdparty, Classpath(creator='javac'))

    # Confirm that we produced classpaths for the managed jars.
    managed_jars = [self.managed_guava, self.managed_hadoop]
    self.assert_select_for_subjects(walk, Select(Classpath), [self.consumes_managed_thirdparty])
    self.assert_select_for_subjects(walk, Select(Classpath), managed_jars,
                                    variants={'resolve': 'latest-hadoop'})

    # Confirm that the produced jars had the appropriate versions.
    self.assertEquals({Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'),
                       Jar('com.google.guava', 'guava', '18.0')},
                      {ret.value for node, ret in walk
                       if node.product == Jar})

  def test_dependency_inference(self):
    """Scala dependency inference introduces dependencies that do not exist in BUILD files."""
    build_request = self.request(['compile'], self.inferred_deps)
    root, = self.build(build_request)
    self.assert_root(root, self.inferred_deps, Classpath(creator='scalac'))

  @unittest.skip('Skipped to expedite landing #3821; see: #4027.')
  def test_dependency_inference_internal(self):
    """Scala dependency inference introduces dependencies that do not exist in BUILD files."""
    build_request = self.request(['compile'], self.inferred_deps)
    root, = self.build(build_request)

    # Confirm that we requested a classpath for the root and inferred targets.
    self.assert_select_for_subjects(walk, Select(Classpath), [self.inferred_deps, self.java_simple])

  def test_multiple_classpath_entries(self):
    """Multiple Classpath products for a single subject currently cause a failure."""
    build_request = self.request(['compile'], self.java_multi)
    root, = self.build(build_request)

    # Validate that the root failed.
    self.assert_root_failed(root, self.java_multi, "Conflicting values produced for")

  def test_descendant_specs(self):
    """Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory."""
    spec = self.spec_parser.parse_spec('3rdparty/jvm::')
    selector = Select(BuildFileAddresses)
    build_request = self.scheduler.selection_request([(selector, spec)])
    ((subject, _), root), = self.build(build_request)

    # Validate the root.
    self.assertEqual(spec, subject)
    self.assertEqual(BuildFileAddresses, type(root.value))

    # Confirm that a few expected addresses are in the list.
    self.assertIn(self.guava, root.value.dependencies)
    self.assertIn(self.managed_guava, root.value.dependencies)
    self.assertIn(self.managed_resolve_latest, root.value.dependencies)

  def test_sibling_specs(self):
    """Test that sibling Addresses are parsed in the 3rdparty/jvm directory."""
    spec = self.spec_parser.parse_spec('3rdparty/jvm:')
    selector = Select(BuildFileAddresses)
    build_request = self.scheduler.selection_request([(selector,spec)])
    ((subject, _), root), = self.build(build_request)

    # Validate the root.
    self.assertEqual(spec, subject)
    self.assertEqual(BuildFileAddresses, type(root.value))

    # Confirm that an expected address is in the list.
    self.assertIn(self.guava, root.value.dependencies)
    # And that a subdirectory address is not.
    self.assertNotIn(self.managed_guava, root.value.dependencies)

  def test_scheduler_visualize(self):
    spec = self.spec_parser.parse_spec('3rdparty/jvm:')
    build_request = self.request(['list'], spec)
    self.build(build_request)

    with temporary_dir() as td:
      output_path = os.path.join(td, 'output.dot')
      self.scheduler.visualize_graph_to_file(output_path)
      with open(output_path, 'rb') as fh:
        graphviz_output = fh.read().strip()

    self.assertIn('digraph', graphviz_output)
    self.assertIn(' -> ', graphviz_output)
Beispiel #4
0
class SchedulerTest(unittest.TestCase):
    def setUp(self):
        build_root = os.path.join(os.path.dirname(__file__), 'examples',
                                  'scheduler_inputs')
        self.spec_parser = CmdLineSpecParser(build_root)
        self.scheduler = setup_json_scheduler(build_root, inline_nodes=False)
        self.pg = self.scheduler.product_graph
        self.engine = LocalSerialEngine(self.scheduler)

        self.guava = Address.parse('3rdparty/jvm:guava')
        self.thrift = Address.parse('src/thrift/codegen/simple')
        self.java = Address.parse('src/java/codegen/simple')
        self.java_simple = Address.parse('src/java/simple')
        self.java_multi = Address.parse('src/java/multiple_classpath_entries')
        self.no_variant_thrift = Address.parse(
            'src/java/codegen/selector:conflict')
        self.unconfigured_thrift = Address.parse(
            'src/thrift/codegen/unconfigured')
        self.resources = Address.parse('src/resources/simple')
        self.consumes_resources = Address.parse('src/java/consumes_resources')
        self.consumes_managed_thirdparty = Address.parse(
            'src/java/managed_thirdparty')
        self.managed_guava = Address.parse('3rdparty/jvm/managed:guava')
        self.managed_hadoop = Address.parse(
            '3rdparty/jvm/managed:hadoop-common')
        self.managed_resolve_latest = Address.parse(
            '3rdparty/jvm/managed:latest-hadoop')
        self.inferred_deps = Address.parse('src/scala/inferred_deps')

    def assert_select_for_subjects(self,
                                   walk,
                                   selector,
                                   subjects,
                                   variants=None):
        node_type = SelectNode

        variants = tuple(variants.items()) if variants else None
        self.assertEqual(
            {node_type(subject, variants, selector)
             for subject in subjects}, {
                 node
                 for node, _ in walk if node.product == selector.product
                 and isinstance(node, node_type) and node.variants == variants
             })

    def build_and_walk(self, build_request):
        """Build and then walk the given build_request, returning the walked graph as a list."""
        result = self.engine.execute(build_request)
        self.assertIsNone(result.error)
        return list(self.scheduler.product_graph.walk(build_request.roots))

    def request(self, goals, *addresses):
        return self.request_specs(
            goals, *[self.spec_parser.parse_spec(str(a)) for a in addresses])

    def request_specs(self, goals, *specs):
        return self.scheduler.build_request(goals=goals, subjects=specs)

    def assert_resolve_only(self, goals, root_specs, jars):
        build_request = self.request(goals, *root_specs)
        walk = self.build_and_walk(build_request)

        # Expect a SelectNode for each of the Jar/Classpath.
        self.assert_select_for_subjects(walk, Select(Jar), jars)
        self.assert_select_for_subjects(walk, Select(Classpath), jars)

    def assert_root(self, walk, node, return_value):
        """Asserts that the first Node in a walk was a DependenciesNode with the single given result."""
        root, root_state = walk[0]
        self.assertEquals(type(root), DependenciesNode)
        self.assertEquals(Return([return_value]), root_state)
        self.assertIn(
            (node, Return(return_value)),
            [(d, self.pg.state(d)) for d in self.pg.dependencies_of(root)])

    def assert_root_failed(self, walk, node, thrown_type):
        """Asserts that the first Node in a walk was a DependenciesNode with a Throw result."""
        root, root_state = walk[0]
        self.assertEquals(type(root), DependenciesNode)
        self.assertEquals(Throw, type(root_state))
        dependencies = [(d, self.pg.state(d))
                        for d in self.pg.dependencies_of(root)]
        self.assertIn(
            (node, thrown_type),
            [(k, type(v.exc)) for k, v in dependencies if type(v) is Throw])

    def test_type_error_on_unexpected_subject_type(self):
        with self.assertRaises(TypeError) as cm:
            self.scheduler.build_request(goals={}, subjects=['string'])
        self.assertEquals(
            "Unsupported root subject type: <type 'unicode'> for u'string'",
            str(cm.exception))

    def test_resolve(self):
        self.assert_resolve_only(goals=['resolve'],
                                 root_specs=['3rdparty/jvm:guava'],
                                 jars=[self.guava])

    def test_compile_only_3rdparty(self):
        self.assert_resolve_only(goals=['compile'],
                                 root_specs=['3rdparty/jvm:guava'],
                                 jars=[self.guava])

    def test_gen_noop(self):
        # TODO(John Sirois): Ask around - is this OK?
        # This is different than today.  There is a gen'able target reachable from the java target, but
        # the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's
        # handed is not thrift.
        build_request = self.request(['gen'], self.java)
        walk = self.build_and_walk(build_request)

        self.assert_select_for_subjects(walk, Select(JavaSources,
                                                     optional=True),
                                        [self.java])

    def test_gen(self):
        build_request = self.request(['gen'], self.thrift)
        walk = self.build_and_walk(build_request)

        # Root: expect the synthetic GenGoal product.
        self.assert_root(
            walk, SelectNode(self.thrift, None, Select(GenGoal)),
            GenGoal("non-empty input to satisfy the Goal constructor"))

        variants = {'thrift': 'apache_java'}
        # Expect ThriftSources to have been selected.
        self.assert_select_for_subjects(walk,
                                        Select(ThriftSources), [self.thrift],
                                        variants=variants)
        # Expect an ApacheThriftJavaConfiguration to have been used via the default Variants.
        self.assert_select_for_subjects(walk,
                                        SelectVariant(
                                            ApacheThriftJavaConfiguration,
                                            variant_key='thrift'),
                                        [self.thrift],
                                        variants=variants)

    def test_codegen_simple(self):
        build_request = self.request(['compile'], self.java)
        walk = self.build_and_walk(build_request)

        # The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants.
        subjects = [self.guava, self.java, self.thrift]
        variant_subjects = [
            Jar(org='org.apache.thrift',
                name='libthrift',
                rev='0.9.2',
                type_alias='jar'),
            Jar(org='commons-lang',
                name='commons-lang',
                rev='2.5',
                type_alias='jar'),
            Address.parse('src/thrift:slf4j-api')
        ]

        # Root: expect a DependenciesNode depending on a SelectNode with compilation via javac.
        self.assert_root(walk, SelectNode(self.java, None, Select(Classpath)),
                         Classpath(creator='javac'))

        # Confirm that exactly the expected subjects got Classpaths.
        self.assert_select_for_subjects(walk, Select(Classpath), subjects)
        self.assert_select_for_subjects(walk,
                                        Select(Classpath),
                                        variant_subjects,
                                        variants={'thrift': 'apache_java'})

    def test_consumes_resources(self):
        build_request = self.request(['compile'], self.consumes_resources)
        walk = self.build_and_walk(build_request)

        # Validate the root.
        self.assert_root(
            walk, SelectNode(self.consumes_resources, None, Select(Classpath)),
            Classpath(creator='javac'))

        # Confirm a classpath for the resources target and other subjects. We know that they are
        # reachable from the root (since it was involved in this walk).
        subjects = [self.resources, self.consumes_resources, self.guava]
        self.assert_select_for_subjects(walk, Select(Classpath), subjects)

    def test_managed_resolve(self):
        """A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars."""
        build_request = self.request(['compile'],
                                     self.consumes_managed_thirdparty)
        walk = self.build_and_walk(build_request)

        # Validate the root.
        self.assert_root(
            walk,
            SelectNode(self.consumes_managed_thirdparty, None,
                       Select(Classpath)), Classpath(creator='javac'))

        # Confirm that we produced classpaths for the managed jars.
        managed_jars = [self.managed_guava, self.managed_hadoop]
        self.assert_select_for_subjects(walk, Select(Classpath),
                                        [self.consumes_managed_thirdparty])
        self.assert_select_for_subjects(walk,
                                        Select(Classpath),
                                        managed_jars,
                                        variants={'resolve': 'latest-hadoop'})

        # Confirm that the produced jars had the appropriate versions.
        self.assertEquals(
            {
                Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'),
                Jar('com.google.guava', 'guava', '18.0')
            }, {
                ret.value
                for node, ret in walk
                if node.product == Jar and isinstance(node, SelectNode)
            })

    def test_dependency_inference(self):
        """Scala dependency inference introduces dependencies that do not exist in BUILD files."""
        build_request = self.request(['compile'], self.inferred_deps)
        walk = self.build_and_walk(build_request)

        # Validate the root.
        self.assert_root(
            walk, SelectNode(self.inferred_deps, None, Select(Classpath)),
            Classpath(creator='scalac'))

        # Confirm that we requested a classpath for the root and inferred targets.
        self.assert_select_for_subjects(walk, Select(Classpath),
                                        [self.inferred_deps, self.java_simple])

    def test_multiple_classpath_entries(self):
        """Multiple Classpath products for a single subject currently cause a failure."""
        build_request = self.request(['compile'], self.java_multi)
        walk = self.build_and_walk(build_request)

        # Validate that the root failed.
        self.assert_root_failed(
            walk, SelectNode(self.java_multi, None, Select(Classpath)),
            ConflictingProducersError)

    def test_descendant_specs(self):
        """Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory."""
        spec = self.spec_parser.parse_spec('3rdparty/jvm::')
        build_request = self.request_specs(['list'], spec)
        walk = self.build_and_walk(build_request)

        # Validate the root.
        root, root_state = walk[0]
        root_value = root_state.value
        self.assertEqual(
            DependenciesNode(
                spec, None,
                SelectDependencies(Address, Addresses,
                                   field_types=(Address, ))), root)
        self.assertEqual(list, type(root_value))

        # Confirm that a few expected addresses are in the list.
        self.assertIn(self.guava, root_value)
        self.assertIn(self.managed_guava, root_value)
        self.assertIn(self.managed_resolve_latest, root_value)

    def test_sibling_specs(self):
        """Test that sibling Addresses are parsed in the 3rdparty/jvm directory."""
        spec = self.spec_parser.parse_spec('3rdparty/jvm:')
        build_request = self.request_specs(['list'], spec)
        walk = self.build_and_walk(build_request)

        # Validate the root.
        root, root_state = walk[0]
        root_value = root_state.value
        self.assertEqual(
            DependenciesNode(
                spec, None,
                SelectDependencies(Address, Addresses,
                                   field_types=(Address, ))), root)
        self.assertEqual(list, type(root_value))

        # Confirm that an expected address is in the list.
        self.assertIn(self.guava, root_value)
        # And that an subdirectory address is not.
        self.assertNotIn(self.managed_guava, root_value)

    def test_scheduler_visualize(self):
        spec = self.spec_parser.parse_spec('3rdparty/jvm:')
        build_request = self.request_specs(['list'], spec)
        self.build_and_walk(build_request)

        graphviz_output = '\n'.join(
            self.scheduler.product_graph.visualize(build_request.roots))

        with temporary_dir() as td:
            output_path = os.path.join(td, 'output.dot')
            self.scheduler.visualize_graph_to_file(build_request.roots,
                                                   output_path)
            with open(output_path, 'rb') as fh:
                graphviz_disk_output = fh.read().strip()

        self.assertEqual(graphviz_output, graphviz_disk_output)
        self.assertIn('digraph', graphviz_output)
        self.assertIn(' -> ', graphviz_output)
Beispiel #5
0
class SchedulerTest(unittest.TestCase):
  def setUp(self):
    build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs')
    self.spec_parser = CmdLineSpecParser(build_root)
    self.scheduler = setup_json_scheduler(build_root, inline_nodes=False)
    self.pg = self.scheduler.product_graph
    self.engine = LocalSerialEngine(self.scheduler)

    self.guava = Address.parse('3rdparty/jvm:guava')
    self.thrift = Address.parse('src/thrift/codegen/simple')
    self.java = Address.parse('src/java/codegen/simple')
    self.java_simple = Address.parse('src/java/simple')
    self.java_multi = Address.parse('src/java/multiple_classpath_entries')
    self.no_variant_thrift = Address.parse('src/java/codegen/selector:conflict')
    self.unconfigured_thrift = Address.parse('src/thrift/codegen/unconfigured')
    self.resources = Address.parse('src/resources/simple')
    self.consumes_resources = Address.parse('src/java/consumes_resources')
    self.consumes_managed_thirdparty = Address.parse('src/java/managed_thirdparty')
    self.managed_guava = Address.parse('3rdparty/jvm/managed:guava')
    self.managed_hadoop = Address.parse('3rdparty/jvm/managed:hadoop-common')
    self.managed_resolve_latest = Address.parse('3rdparty/jvm/managed:latest-hadoop')
    self.inferred_deps = Address.parse('src/scala/inferred_deps')

  def assert_select_for_subjects(self, walk, selector, subjects, variants=None):
    node_type = SelectNode

    variants = tuple(variants.items()) if variants else None
    self.assertEqual({node_type(subject, variants, selector) for subject in subjects},
                     {node for node, _ in walk
                      if node.product == selector.product and isinstance(node, node_type) and node.variants == variants})

  def build_and_walk(self, build_request):
    """Build and then walk the given build_request, returning the walked graph as a list."""
    result = self.engine.execute(build_request)
    self.assertIsNone(result.error)
    return list(self.scheduler.product_graph.walk(build_request.roots))

  def request(self, goals, *addresses):
    return self.request_specs(goals, *[self.spec_parser.parse_spec(str(a)) for a in addresses])

  def request_specs(self, goals, *specs):
    return self.scheduler.build_request(goals=goals, subjects=specs)

  def assert_resolve_only(self, goals, root_specs, jars):
    build_request = self.request(goals, *root_specs)
    walk = self.build_and_walk(build_request)

    # Expect a SelectNode for each of the Jar/Classpath.
    self.assert_select_for_subjects(walk, Select(Jar), jars)
    self.assert_select_for_subjects(walk, Select(Classpath), jars)

  def assert_root(self, walk, node, return_value):
    """Asserts that the first Node in a walk was a DependenciesNode with the single given result."""
    root, root_state = walk[0]
    self.assertEquals(type(root), DependenciesNode)
    self.assertEquals(Return([return_value]), root_state)
    self.assertIn((node, Return(return_value)),
                  [(d, self.pg.state(d)) for d in self.pg.dependencies_of(root)])

  def assert_root_failed(self, walk, node, thrown_type):
    """Asserts that the first Node in a walk was a DependenciesNode with a Throw result."""
    root, root_state = walk[0]
    self.assertEquals(type(root), DependenciesNode)
    self.assertEquals(Throw, type(root_state))
    dependencies = [(d, self.pg.state(d)) for d in self.pg.dependencies_of(root)]
    self.assertIn((node, thrown_type), [(k, type(v.exc))
                                        for k, v in dependencies if type(v) is Throw])

  def test_resolve(self):
    self.assert_resolve_only(goals=['resolve'],
                             root_specs=['3rdparty/jvm:guava'],
                             jars=[self.guava])

  def test_compile_only_3rdparty(self):
    self.assert_resolve_only(goals=['compile'],
                             root_specs=['3rdparty/jvm:guava'],
                             jars=[self.guava])

  def test_gen_noop(self):
    # TODO(John Sirois): Ask around - is this OK?
    # This is different than today.  There is a gen'able target reachable from the java target, but
    # the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's
    # handed is not thrift.
    build_request = self.request(['gen'], self.java)
    walk = self.build_and_walk(build_request)

    self.assert_select_for_subjects(walk, Select(JavaSources, optional=True), [self.java])

  def test_gen(self):
    build_request = self.request(['gen'], self.thrift)
    walk = self.build_and_walk(build_request)

    # Root: expect the synthetic GenGoal product.
    self.assert_root(walk,
                     SelectNode(self.thrift, None, Select(GenGoal)),
                     GenGoal("non-empty input to satisfy the Goal constructor"))

    variants = {'thrift': 'apache_java'}
    # Expect ThriftSources to have been selected.
    self.assert_select_for_subjects(walk, Select(ThriftSources), [self.thrift], variants=variants)
    # Expect an ApacheThriftJavaConfiguration to have been used via the default Variants.
    self.assert_select_for_subjects(walk, SelectVariant(ApacheThriftJavaConfiguration,
                                                        variant_key='thrift'),
                                    [self.thrift],
                                    variants=variants)

  def test_codegen_simple(self):
    build_request = self.request(['compile'], self.java)
    walk = self.build_and_walk(build_request)

    # The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants.
    subjects = [
        self.guava,
        self.java,
        self.thrift]
    variant_subjects = [
        Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2', type_alias='jar'),
        Jar(org='commons-lang', name='commons-lang', rev='2.5', type_alias='jar'),
        Address.parse('src/thrift:slf4j-api')]

    # Root: expect a DependenciesNode depending on a SelectNode with compilation via javac.
    self.assert_root(walk,
                     SelectNode(self.java, None, Select(Classpath)),
                     Classpath(creator='javac'))

    # Confirm that exactly the expected subjects got Classpaths.
    self.assert_select_for_subjects(walk, Select(Classpath), subjects)
    self.assert_select_for_subjects(walk, Select(Classpath), variant_subjects,
                                    variants={'thrift': 'apache_java'})

  def test_consumes_resources(self):
    build_request = self.request(['compile'], self.consumes_resources)
    walk = self.build_and_walk(build_request)

    # Validate the root.
    self.assert_root(walk,
                     SelectNode(self.consumes_resources, None, Select(Classpath)),
                     Classpath(creator='javac'))

    # Confirm a classpath for the resources target and other subjects. We know that they are
    # reachable from the root (since it was involved in this walk).
    subjects = [self.resources,
                self.consumes_resources,
                self.guava]
    self.assert_select_for_subjects(walk, Select(Classpath), subjects)

  def test_managed_resolve(self):
    """A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars."""
    build_request = self.request(['compile'], self.consumes_managed_thirdparty)
    walk = self.build_and_walk(build_request)

    # Validate the root.
    self.assert_root(walk,
                     SelectNode(self.consumes_managed_thirdparty, None, Select(Classpath)),
                     Classpath(creator='javac'))

    # Confirm that we produced classpaths for the managed jars.
    managed_jars = [self.managed_guava,
                    self.managed_hadoop]
    self.assert_select_for_subjects(walk, Select(Classpath), [self.consumes_managed_thirdparty])
    self.assert_select_for_subjects(walk, Select(Classpath), managed_jars,
                                    variants={'resolve': 'latest-hadoop'})

    # Confirm that the produced jars had the appropriate versions.
    self.assertEquals({Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'),
                       Jar('com.google.guava', 'guava', '18.0')},
                      {ret.value for node, ret in walk
                       if node.product == Jar and isinstance(node, SelectNode)})

  def test_dependency_inference(self):
    """Scala dependency inference introduces dependencies that do not exist in BUILD files."""
    build_request = self.request(['compile'], self.inferred_deps)
    walk = self.build_and_walk(build_request)

    # Validate the root.
    self.assert_root(walk,
                     SelectNode(self.inferred_deps, None, Select(Classpath)),
                     Classpath(creator='scalac'))

    # Confirm that we requested a classpath for the root and inferred targets.
    self.assert_select_for_subjects(walk, Select(Classpath), [self.inferred_deps, self.java_simple])

  def test_multiple_classpath_entries(self):
    """Multiple Classpath products for a single subject currently cause a failure."""
    build_request = self.request(['compile'], self.java_multi)
    walk = self.build_and_walk(build_request)

    # Validate that the root failed.
    self.assert_root_failed(walk,
                            SelectNode(self.java_multi, None, Select(Classpath)),
                            ConflictingProducersError)

  def test_descendant_specs(self):
    """Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory."""
    spec = self.spec_parser.parse_spec('3rdparty/jvm::')
    build_request = self.request_specs(['list'], spec)
    walk = self.build_and_walk(build_request)

    # Validate the root.
    root, root_state = walk[0]
    root_value = root_state.value
    self.assertEqual(DependenciesNode(spec, None, SelectDependencies(Address, Addresses)), root)
    self.assertEqual(list, type(root_value))

    # Confirm that a few expected addresses are in the list.
    self.assertIn(self.guava, root_value)
    self.assertIn(self.managed_guava, root_value)
    self.assertIn(self.managed_resolve_latest, root_value)

  def test_sibling_specs(self):
    """Test that sibling Addresses are parsed in the 3rdparty/jvm directory."""
    spec = self.spec_parser.parse_spec('3rdparty/jvm:')
    build_request = self.request_specs(['list'], spec)
    walk = self.build_and_walk(build_request)

    # Validate the root.
    root, root_state = walk[0]
    root_value = root_state.value
    self.assertEqual(DependenciesNode(spec, None, SelectDependencies(Address, Addresses)), root)
    self.assertEqual(list, type(root_value))

    # Confirm that an expected address is in the list.
    self.assertIn(self.guava, root_value)
    # And that an subdirectory address is not.
    self.assertNotIn(self.managed_guava, root_value)

  def test_scheduler_visualize(self):
    spec = self.spec_parser.parse_spec('3rdparty/jvm:')
    build_request = self.request_specs(['list'], spec)
    self.build_and_walk(build_request)

    graphviz_output = '\n'.join(self.scheduler.product_graph.visualize(build_request.roots))

    with temporary_dir() as td:
      output_path = os.path.join(td, 'output.dot')
      self.scheduler.visualize_graph_to_file(build_request.roots, output_path)
      with open(output_path, 'rb') as fh:
        graphviz_disk_output = fh.read().strip()

    self.assertEqual(graphviz_output, graphviz_disk_output)
    self.assertIn('digraph', graphviz_output)
    self.assertIn(' -> ', graphviz_output)
Beispiel #6
0
class SchedulerTest(unittest.TestCase):

    _native = init_native()

    def setUp(self):
        build_root = os.path.join(os.path.dirname(__file__), 'examples',
                                  'scheduler_inputs')
        self.spec_parser = CmdLineSpecParser(build_root)
        self.scheduler = setup_json_scheduler(build_root, self._native)
        self.engine = LocalSerialEngine(self.scheduler)

        self.guava = Address.parse('3rdparty/jvm:guava')
        self.thrift = Address.parse('src/thrift/codegen/simple')
        self.java = Address.parse('src/java/codegen/simple')
        self.java_simple = Address.parse('src/java/simple')
        self.java_multi = Address.parse('src/java/multiple_classpath_entries')
        self.no_variant_thrift = Address.parse(
            'src/java/codegen/selector:conflict')
        self.unconfigured_thrift = Address.parse(
            'src/thrift/codegen/unconfigured')
        self.resources = Address.parse('src/resources/simple')
        self.consumes_resources = Address.parse('src/java/consumes_resources')
        self.consumes_managed_thirdparty = Address.parse(
            'src/java/managed_thirdparty')
        self.managed_guava = Address.parse('3rdparty/jvm/managed:guava')
        self.managed_hadoop = Address.parse(
            '3rdparty/jvm/managed:hadoop-common')
        self.managed_resolve_latest = Address.parse(
            '3rdparty/jvm/managed:latest-hadoop')
        self.inferred_deps = Address.parse('src/scala/inferred_deps')

    def assert_select_for_subjects(self,
                                   walk,
                                   selector,
                                   subjects,
                                   variants=None):
        raise ValueError(walk)

    def build(self, build_request):
        """Execute the given request and return roots as a list of ((subject, product), value) tuples."""
        result = self.engine.execute(build_request)
        self.assertIsNone(result.error)
        return self.scheduler.root_entries(build_request).items()

    def request(self, goals, *subjects):
        return self.scheduler.build_request(goals=goals, subjects=subjects)

    def assert_root(self, root, subject, return_value):
        """Asserts that the given root has the given result."""
        self.assertEquals(subject, root[0][0])
        self.assertEquals(Return(return_value), root[1])

    def assert_root_failed(self, root, subject, msg_str):
        """Asserts that the root was a Throw result containing the given msg string."""
        self.assertEquals(subject, root[0][0])
        self.assertEquals(Throw, type(root[1]))
        self.assertIn(msg_str, str(root[1].exc))

    def test_compile_only_3rdparty(self):
        build_request = self.request(['compile'], self.guava)
        root, = self.build(build_request)
        self.assert_root(root, self.guava, Classpath(creator='ivy_resolve'))

    @unittest.skip('Skipped to expedite landing #3821; see: #4027.')
    def test_compile_only_3rdparty_internal(self):
        build_request = self.request(['compile'], '3rdparty/jvm:guava')
        root, = self.build(build_request)

        # Expect a SelectNode for each of the Jar/Classpath.
        self.assert_select_for_subjects(walk, Select(Jar), [self.guava])
        self.assert_select_for_subjects(walk, Select(Classpath), [self.guava])

    @unittest.skip('Skipped to expedite landing #3821; see: #4020.')
    def test_gen(self):
        build_request = self.request(['gen'], self.thrift)
        root, = self.build(build_request)

        # Root: expect the synthetic GenGoal product.
        self.assert_root(
            root, self.thrift,
            GenGoal("non-empty input to satisfy the Goal constructor"))

        variants = {'thrift': 'apache_java'}
        # Expect ThriftSources to have been selected.
        self.assert_select_for_subjects(walk,
                                        Select(ThriftSources), [self.thrift],
                                        variants=variants)
        # Expect an ApacheThriftJavaConfiguration to have been used via the default Variants.
        self.assert_select_for_subjects(walk,
                                        SelectVariant(
                                            ApacheThriftJavaConfiguration,
                                            variant_key='thrift'),
                                        [self.thrift],
                                        variants=variants)

    @unittest.skip('Skipped to expedite landing #3821; see: #4020.')
    def test_codegen_simple(self):
        build_request = self.request(['compile'], self.java)
        root, = self.build(build_request)

        # The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants.
        subjects = [self.guava, self.java, self.thrift]
        variant_subjects = [
            Jar(org='org.apache.thrift',
                name='libthrift',
                rev='0.9.2',
                type_alias='jar'),
            Jar(org='commons-lang',
                name='commons-lang',
                rev='2.5',
                type_alias='jar'),
            Address.parse('src/thrift:slf4j-api')
        ]

        # Root: expect a DependenciesNode depending on a SelectNode with compilation via javac.
        self.assert_root(root, self.java, Classpath(creator='javac'))

        # Confirm that exactly the expected subjects got Classpaths.
        self.assert_select_for_subjects(walk, Select(Classpath), subjects)
        self.assert_select_for_subjects(walk,
                                        Select(Classpath),
                                        variant_subjects,
                                        variants={'thrift': 'apache_java'})

    def test_consumes_resources(self):
        build_request = self.request(['compile'], self.consumes_resources)
        root, = self.build(build_request)
        self.assert_root(root, self.consumes_resources,
                         Classpath(creator='javac'))

    @unittest.skip('Skipped to expedite landing #3821; see: #4027.')
    def test_consumes_resources_internal(self):
        build_request = self.request(['compile'], self.consumes_resources)
        root, = self.build(build_request)

        # Confirm a classpath for the resources target and other subjects. We know that they are
        # reachable from the root (since it was involved in this walk).
        subjects = [self.resources, self.consumes_resources, self.guava]
        self.assert_select_for_subjects(walk, Select(Classpath), subjects)

    @unittest.skip('Skipped to expedite landing #3821; see: #4020.')
    def test_managed_resolve(self):
        """A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars."""
        build_request = self.request(['compile'],
                                     self.consumes_managed_thirdparty)
        root, = self.build(build_request)

        # Validate the root.
        self.assert_root(root, self.consumes_managed_thirdparty,
                         Classpath(creator='javac'))

        # Confirm that we produced classpaths for the managed jars.
        managed_jars = [self.managed_guava, self.managed_hadoop]
        self.assert_select_for_subjects(walk, Select(Classpath),
                                        [self.consumes_managed_thirdparty])
        self.assert_select_for_subjects(walk,
                                        Select(Classpath),
                                        managed_jars,
                                        variants={'resolve': 'latest-hadoop'})

        # Confirm that the produced jars had the appropriate versions.
        self.assertEquals(
            {
                Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'),
                Jar('com.google.guava', 'guava', '18.0')
            }, {ret.value
                for node, ret in walk if node.product == Jar})

    def test_dependency_inference(self):
        """Scala dependency inference introduces dependencies that do not exist in BUILD files."""
        build_request = self.request(['compile'], self.inferred_deps)
        root, = self.build(build_request)
        self.assert_root(root, self.inferred_deps, Classpath(creator='scalac'))

    @unittest.skip('Skipped to expedite landing #3821; see: #4027.')
    def test_dependency_inference_internal(self):
        """Scala dependency inference introduces dependencies that do not exist in BUILD files."""
        build_request = self.request(['compile'], self.inferred_deps)
        root, = self.build(build_request)

        # Confirm that we requested a classpath for the root and inferred targets.
        self.assert_select_for_subjects(walk, Select(Classpath),
                                        [self.inferred_deps, self.java_simple])

    def test_multiple_classpath_entries(self):
        """Multiple Classpath products for a single subject currently cause a failure."""
        build_request = self.request(['compile'], self.java_multi)
        root, = self.build(build_request)

        # Validate that the root failed.
        self.assert_root_failed(root, self.java_multi,
                                "Conflicting values produced for")

    def test_descendant_specs(self):
        """Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory."""
        spec = self.spec_parser.parse_spec('3rdparty/jvm::')
        selector = SelectDependencies(Address,
                                      Addresses,
                                      field_types=(Address, ))
        build_request = self.scheduler.selection_request([(selector, spec)])
        ((subject, _), root), = self.build(build_request)

        # Validate the root.
        self.assertEqual(spec, subject)
        self.assertEqual(tuple, type(root.value))

        # Confirm that a few expected addresses are in the list.
        self.assertIn(self.guava, root.value)
        self.assertIn(self.managed_guava, root.value)
        self.assertIn(self.managed_resolve_latest, root.value)

    def test_sibling_specs(self):
        """Test that sibling Addresses are parsed in the 3rdparty/jvm directory."""
        spec = self.spec_parser.parse_spec('3rdparty/jvm:')
        selector = SelectDependencies(Address,
                                      Addresses,
                                      field_types=(Address, ))
        build_request = self.scheduler.selection_request([(selector, spec)])
        ((subject, _), root), = self.build(build_request)

        # Validate the root.
        self.assertEqual(spec, subject)
        self.assertEqual(tuple, type(root.value))

        # Confirm that an expected address is in the list.
        self.assertIn(self.guava, root.value)
        # And that a subdirectory address is not.
        self.assertNotIn(self.managed_guava, root.value)

    def test_scheduler_visualize(self):
        spec = self.spec_parser.parse_spec('3rdparty/jvm:')
        build_request = self.request(['list'], spec)
        self.build(build_request)

        with temporary_dir() as td:
            output_path = os.path.join(td, 'output.dot')
            self.scheduler.visualize_graph_to_file(output_path)
            with open(output_path, 'rb') as fh:
                graphviz_output = fh.read().strip()

        self.assertIn('digraph', graphviz_output)
        self.assertIn(' -> ', graphviz_output)