Exemplo n.º 1
0
 def _local_jvm_distribution(settings=None):
   settings_args = [settings] if settings else []
   try:
     local_distribution = JvmPlatform.preferred_jvm_distribution(settings_args, strict=True)
   except DistributionLocator.Error:
     local_distribution = JvmPlatform.preferred_jvm_distribution(settings_args, strict=False)
   return local_distribution
Exemplo n.º 2
0
  def _get_zinc_arguments(settings):
    """Extracts and formats the zinc arguments given in the jvm platform settings.

    This is responsible for the symbol substitution which replaces $JAVA_HOME with the path to an
    appropriate jvm distribution.

    :param settings: The jvm platform settings from which to extract the arguments.
    :type settings: :class:`JvmPlatformSettings`
    """
    zinc_args = [
      '-C-source', '-C{}'.format(settings.source_level),
      '-C-target', '-C{}'.format(settings.target_level),
    ]
    if settings.args:
      settings_args = settings.args
      if any('$JAVA_HOME' in a for a in settings.args):
        try:
          distribution = JvmPlatform.preferred_jvm_distribution([settings], strict=True)
        except DistributionLocator.Error:
          distribution = JvmPlatform.preferred_jvm_distribution([settings], strict=False)
        logger.debug('Substituting "$JAVA_HOME" with "{}" in jvm-platform args.'
                     .format(distribution.home))
        settings_args = (a.replace('$JAVA_HOME', distribution.home) for a in settings.args)
      zinc_args.extend(settings_args)
    return zinc_args
Exemplo n.º 3
0
  def compile(self, ctx, args, classpath, upstream_analysis,
              settings, fatal_warnings, zinc_file_manager,
              javac_plugin_map, scalac_plugin_map):
    try:
      distribution = JvmPlatform.preferred_jvm_distribution([settings], strict=True)
    except DistributionLocator.Error:
      distribution = JvmPlatform.preferred_jvm_distribution([settings], strict=False)

    javac_cmd = ['{}/bin/javac'.format(distribution.real_home)]

    javac_cmd.extend([
      '-classpath', ':'.join(classpath),
    ])

    if settings.args:
      settings_args = settings.args
      if any('$JAVA_HOME' in a for a in settings.args):
        logger.debug('Substituting "$JAVA_HOME" with "{}" in jvm-platform args.'
                     .format(distribution.home))
        settings_args = (a.replace('$JAVA_HOME', distribution.home) for a in settings.args)
      javac_cmd.extend(settings_args)

    javac_cmd.extend([
      '-d', ctx.classes_dir,
      # TODO: support -release
      '-source', str(settings.source_level),
      '-target', str(settings.target_level),
    ])

    javac_cmd.extend(self._javac_plugin_args(javac_plugin_map))

    javac_cmd.extend(args)

    if fatal_warnings:
      javac_cmd.extend(self.get_options().fatal_warnings_enabled_args)
    else:
      javac_cmd.extend(self.get_options().fatal_warnings_disabled_args)

    with argfile.safe_args(ctx.sources, self.get_options()) as batched_sources:
      javac_cmd.extend(batched_sources)

      with self.context.new_workunit(name='javac',
                                     cmd=' '.join(javac_cmd),
                                     labels=[WorkUnitLabel.COMPILER]) as workunit:
        self.context.log.debug('Executing {}'.format(' '.join(javac_cmd)))
        p = subprocess.Popen(javac_cmd, stdout=workunit.output('stdout'), stderr=workunit.output('stderr'))
        return_code = p.wait()
        workunit.set_outcome(WorkUnit.FAILURE if return_code else WorkUnit.SUCCESS)
        if return_code:
          raise TaskError('javac exited with return code {rc}'.format(rc=return_code))
Exemplo n.º 4
0
  def platform(self):
    """Platform associated with this target.

    :return: The jvm platform object.
    :rtype: JvmPlatformSettings
    """
    return JvmPlatform.global_instance().get_platform_for_target(self)
Exemplo n.º 5
0
  def execute(self):
    if JvmPlatform.global_instance().get_options().compiler != self.compiler_name:
      # If the requested compiler is not the one supported by this task,
      # bail early.
      return

    # In case we have no relevant targets and return early, create the requested product maps.
    self.create_empty_extra_products()

    relevant_targets = list(self.context.targets(predicate=self.select))

    if not relevant_targets:
      return

    # Clone the compile_classpath to the runtime_classpath.
    classpath_product = self.create_runtime_classpath()

    fingerprint_strategy = DependencyContext.global_instance().create_fingerprint_strategy(
        classpath_product)
    # Note, JVM targets are validated (`vts.update()`) as they succeed.  As a result,
    # we begin writing artifacts out to the cache immediately instead of waiting for
    # all targets to finish.
    with self.invalidated(relevant_targets,
                          invalidate_dependents=True,
                          fingerprint_strategy=fingerprint_strategy,
                          topological_order=True) as invalidation_check:

      compile_contexts = {vt.target: self.create_compile_context(vt.target, vt.results_dir)
                          for vt in invalidation_check.all_vts}

      self.do_compile(
        invalidation_check,
        compile_contexts,
        classpath_product,
      )

      if not self.get_options().use_classpath_jars:
        # Once compilation has completed, replace the classpath entry for each target with
        # its jar'd representation.
        for ccs in compile_contexts.values():
          cc = self.select_runtime_context(ccs)
          for conf in self._confs:
            classpath_product.remove_for_target(cc.target, [(conf, cc.classes_dir.path)])
            classpath_product.add_for_target(cc.target, [(conf, cc.jar_file.path)])
Exemplo n.º 6
0
    def test_distribution_from_targets_passes_through_platforms(self):
        fake_dist = "a dist"
        java8_platform = JvmPlatformSettings('8', '8', [])
        targets = [
            self.make_target('platformed_target', JvmTarget, platform='java8')
        ]
        with unittest.mock.patch.object(
                JvmPlatform, 'preferred_jvm_distribution') as plat_mock:
            with unittest.mock.patch.object(JvmPlatform.global_instance(), 'get_platform_for_target') as \
              target_plat_mock:
                target_plat_mock.return_value = java8_platform

                plat_mock.return_value = fake_dist
                dist = self.task.preferred_jvm_distribution_for_targets(
                    targets)

                plat_mock.assert_called_once_with([java8_platform],
                                                  strict=False,
                                                  jdk=False)
                self.assertEqual(fake_dist, dist)
Exemplo n.º 7
0
    def test_jvm_options_from_platform_shlexed(self):
        init_subsystem(
            JvmPlatform,
            options={
                "jvm-platform": {
                    "platforms": {
                        "platform-with-shlexable-vm-options": {
                            "target": "8",
                            "jvm_options": ["-Dsomething -Dsomethingelse"],
                        },
                    },
                }
            },
        )
        instance = JvmPlatform.global_instance()
        need_shlex_options = instance.get_platform_by_name(
            "platform-with-shlexable-vm-options")

        assert ("-Dsomething",
                "-Dsomethingelse") == need_shlex_options.jvm_options
Exemplo n.º 8
0
    def test_distribution_from_targets_passes_through_platforms(self):
        fake_dist = "a dist"
        java8_platform = self.java8_platform()
        targets = [
            self.make_target("platformed_target", JvmTarget, platform="java8")
        ]
        with unittest.mock.patch.object(
                JvmPlatform, "preferred_jvm_distribution") as plat_mock:
            with unittest.mock.patch.object(
                    JvmPlatform.global_instance(),
                    "get_platform_for_target") as target_plat_mock:
                target_plat_mock.return_value = java8_platform

                plat_mock.return_value = fake_dist
                dist = self.task.preferred_jvm_distribution_for_targets(
                    targets)

                plat_mock.assert_called_once_with([java8_platform],
                                                  strict=None,
                                                  jdk=False)
                self.assertEqual(fake_dist, dist)
Exemplo n.º 9
0
    def test_jvm_options(self):
        init_subsystem(
            JvmPlatform,
            options={
                "jvm-platform": {
                    "platforms": {
                        "platform-with-jvm-options": {
                            "target": "8",
                            "jvm_options": ["-Dsomething"],
                        },
                        "platform-without-jvm-options": {"target": "8"},
                    },
                }
            },
        )
        instance = JvmPlatform.global_instance()
        with_options = instance.get_platform_by_name("platform-with-jvm-options")
        without_options = instance.get_platform_by_name("platform-without-jvm-options")

        assert ("-Dsomething",) == with_options.jvm_options
        assert tuple() == without_options.jvm_options
Exemplo n.º 10
0
  def execute(self):
    if self.goal not in JvmPrepCommand.goals():
      raise  AssertionError('Got goal "{}". Expected goal to be one of {}'.format(
          self.goal, JvmPrepCommand.goals()))

    targets = self.context.targets(postorder=True,  predicate=self.runnable_prep_cmd)

    compile_classpath = self.context.products.get_data('compile_classpath')
    classpath_products = self.context.products.get_data('runtime_classpath', compile_classpath.copy)

    with self.context.new_workunit(name='jvm_prep_command', labels=[WorkUnitLabel.PREP]) as workunit:
      for target in targets:
        distribution = JvmPlatform.preferred_jvm_distribution([target.platform])
        executor = SubprocessExecutor(distribution)

        mainclass = target.payload.get_field_value('mainclass')
        args = target.payload.get_field_value('args', [])
        target_jvm_options = target.payload.get_field_value('jvm_options', [])
        cp = list(ClasspathUtil.classpath(target.closure(), classpath_products))
        if not cp:
          raise TaskError('target {} has no classpath. (Add dependencies= parameter?'
                          .format(target.address.spec))
        self.context.log.info('Running prep command for {}'.format(target.address.spec))
        returncode = distribution.execute_java(
          executor=executor,
          classpath=cp,
          main=mainclass,
          jvm_options=target_jvm_options,
          args=args,
          workunit_factory=self.context.new_workunit,
          workunit_name='run',
          workunit_labels=[WorkUnitLabel.PREP],
        )

        workunit.set_outcome(WorkUnit.FAILURE if returncode else WorkUnit.SUCCESS)
        if returncode:
          raise TaskError('RunJvmPrepCommand failed to run {}'.format(mainclass))
Exemplo n.º 11
0
    def run_tests(self, fail_fast, test_targets, output_dir, coverage):
        test_registry = self._collect_test_targets(test_targets)
        if test_registry.empty:
            return TestResult.rc(0)

        coverage.instrument(output_dir)

        def parse_error_handler(parse_error):
            # Just log and move on since the result is only used to characterize failures, and raising
            # an error here would just distract from the underlying test failures.
            self.context.log.error(
                'Error parsing test result file {path}: {cause}'.format(
                    path=parse_error.xml_path, cause=parse_error.cause))

        # The 'instrument_classpath' product below below will be `None` if not set, and we'll default
        # back to runtime_classpath
        classpath_product = self.context.products.get_data(
            'instrument_classpath')

        result = 0
        for batch_id, (properties,
                       batch) in enumerate(self._iter_batches(test_registry)):
            (workdir, platform, target_jvm_options, target_env_vars,
             concurrency, threads) = properties

            batch_output_dir = output_dir
            if self._batched:
                batch_output_dir = os.path.join(batch_output_dir,
                                                'batch-{}'.format(batch_id))

            run_modifications = coverage.run_modifications(batch_output_dir)

            extra_jvm_options = run_modifications.extra_jvm_options

            # Batches of test classes will likely exist within the same targets: dedupe them.
            relevant_targets = {
                test_registry.get_owning_target(t)
                for t in batch
            }

            complete_classpath = OrderedSet()
            complete_classpath.update(run_modifications.classpath_prepend)
            complete_classpath.update(JUnit.global_instance().runner_classpath(
                self.context))
            complete_classpath.update(
                self.classpath(relevant_targets,
                               classpath_product=classpath_product))

            distribution = JvmPlatform.preferred_jvm_distribution(
                [platform], self._strict_jvm_version)

            # Override cmdline args with values from junit_test() target that specify concurrency:
            args = self._args(fail_fast, batch_output_dir) + [u'-xmlreport']

            if concurrency is not None:
                args = remove_arg(args, '-default-parallel')
                if concurrency == JUnitTests.CONCURRENCY_SERIAL:
                    args = ensure_arg(args,
                                      '-default-concurrency',
                                      param='SERIAL')
                elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES:
                    args = ensure_arg(args,
                                      '-default-concurrency',
                                      param='PARALLEL_CLASSES')
                elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_METHODS:
                    args = ensure_arg(args,
                                      '-default-concurrency',
                                      param='PARALLEL_METHODS')
                elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES_AND_METHODS:
                    args = ensure_arg(args,
                                      '-default-concurrency',
                                      param='PARALLEL_CLASSES_AND_METHODS')

            if threads is not None:
                args = remove_arg(args, '-parallel-threads', has_param=True)
                args += ['-parallel-threads', str(threads)]

            batch_test_specs = [test.render_test_spec() for test in batch]
            with argfile.safe_args(batch_test_specs,
                                   self.get_options()) as batch_tests:
                with self._chroot(relevant_targets, workdir) as chroot:
                    self.context.log.debug('CWD = {}'.format(chroot))
                    self.context.log.debug('platform = {}'.format(platform))
                    with environment_as(**dict(target_env_vars)):
                        subprocess_result = self._spawn_and_wait(
                            executor=SubprocessExecutor(distribution),
                            distribution=distribution,
                            classpath=complete_classpath,
                            main=JUnit.RUNNER_MAIN,
                            jvm_options=self.jvm_options + extra_jvm_options +
                            list(target_jvm_options),
                            args=args + batch_tests,
                            workunit_factory=self.context.new_workunit,
                            workunit_name='run',
                            workunit_labels=[WorkUnitLabel.TEST],
                            cwd=chroot,
                            synthetic_jar_dir=batch_output_dir,
                            create_synthetic_jar=self.synthetic_classpath,
                        )
                        self.context.log.debug(
                            'JUnit subprocess exited with result ({})'.format(
                                subprocess_result))
                        result += abs(subprocess_result)

                tests_info = self.parse_test_info(batch_output_dir,
                                                  parse_error_handler,
                                                  ['classname'])
                for test_name, test_info in tests_info.items():
                    test_item = Test(test_info['classname'], test_name)
                    test_target = test_registry.get_owning_target(test_item)
                    self.report_all_info_for_single_test(
                        self.options_scope, test_target, test_name, test_info)

                if result != 0 and fail_fast:
                    break

        if result == 0:
            return TestResult.rc(0)

        target_to_failed_test = parse_failed_targets(test_registry, output_dir,
                                                     parse_error_handler)

        def sort_owning_target(t):
            return t.address.spec if t else None

        failed_targets = sorted(target_to_failed_test, key=sort_owning_target)
        error_message_lines = []
        if self._failure_summary:

            def render_owning_target(t):
                return t.address.reference() if t else '<Unknown Target>'

            for target in failed_targets:
                error_message_lines.append('\n{indent}{owner}'.format(
                    indent=' ' * 4, owner=render_owning_target(target)))
                for test in sorted(target_to_failed_test[target]):
                    error_message_lines.append(
                        '{indent}{classname}#{methodname}'.format(
                            indent=' ' * 8,
                            classname=test.classname,
                            methodname=test.methodname))
        error_message_lines.append(
            '\njava {main} ... exited non-zero ({code}); {failed} failed {targets}.'
            .format(main=JUnit.RUNNER_MAIN,
                    code=result,
                    failed=len(failed_targets),
                    targets=pluralize(len(failed_targets), 'target')))
        return TestResult(msg='\n'.join(error_message_lines),
                          rc=result,
                          failed_targets=failed_targets)
Exemplo n.º 12
0
  def run_tests(self, fail_fast, test_targets, output_dir, coverage):
    test_registry = self._collect_test_targets(test_targets)
    if test_registry.empty:
      return TestResult.rc(0)

    coverage.instrument(output_dir)

    def parse_error_handler(parse_error):
      # Just log and move on since the result is only used to characterize failures, and raising
      # an error here would just distract from the underlying test failures.
      self.context.log.error('Error parsing test result file {path}: {cause}'
                             .format(path=parse_error.xml_path, cause=parse_error.cause))

    # The 'instrument_classpath' product below below will be `None` if not set, and we'll default
    # back to runtime_classpath
    classpath_product = self.context.products.get_data('instrument_classpath')

    result = 0
    for batch_id, (properties, batch) in enumerate(self._iter_batches(test_registry)):
      (workdir, platform, target_jvm_options, target_env_vars, concurrency, threads) = properties

      batch_output_dir = output_dir
      if self._batched:
        batch_output_dir = os.path.join(batch_output_dir, 'batch-{}'.format(batch_id))

      run_modifications = coverage.run_modifications(batch_output_dir)

      extra_jvm_options = run_modifications.extra_jvm_options

      # Batches of test classes will likely exist within the same targets: dedupe them.
      relevant_targets = {test_registry.get_owning_target(t) for t in batch}

      complete_classpath = OrderedSet()
      complete_classpath.update(run_modifications.classpath_prepend)
      complete_classpath.update(JUnit.global_instance().runner_classpath(self.context))
      complete_classpath.update(self.classpath(relevant_targets,
                                               classpath_product=classpath_product))

      distribution = JvmPlatform.preferred_jvm_distribution([platform], self._strict_jvm_version)

      # Override cmdline args with values from junit_test() target that specify concurrency:
      args = self._args(fail_fast, batch_output_dir) + [u'-xmlreport']

      if concurrency is not None:
        args = remove_arg(args, '-default-parallel')
        if concurrency == JUnitTests.CONCURRENCY_SERIAL:
          args = ensure_arg(args, '-default-concurrency', param='SERIAL')
        elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES:
          args = ensure_arg(args, '-default-concurrency', param='PARALLEL_CLASSES')
        elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_METHODS:
          args = ensure_arg(args, '-default-concurrency', param='PARALLEL_METHODS')
        elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES_AND_METHODS:
          args = ensure_arg(args, '-default-concurrency', param='PARALLEL_CLASSES_AND_METHODS')

      if threads is not None:
        args = remove_arg(args, '-parallel-threads', has_param=True)
        args += ['-parallel-threads', str(threads)]

      batch_test_specs = [test.render_test_spec() for test in batch]
      with argfile.safe_args(batch_test_specs, self.get_options()) as batch_tests:
        with self._chroot(relevant_targets, workdir) as chroot:
          self.context.log.debug('CWD = {}'.format(chroot))
          self.context.log.debug('platform = {}'.format(platform))
          with environment_as(**dict(target_env_vars)):
            subprocess_result = self._spawn_and_wait(
              executor=SubprocessExecutor(distribution),
              distribution=distribution,
              classpath=complete_classpath,
              main=JUnit.RUNNER_MAIN,
              jvm_options=self.jvm_options + extra_jvm_options + list(target_jvm_options),
              args=args + batch_tests,
              workunit_factory=self.context.new_workunit,
              workunit_name='run',
              workunit_labels=[WorkUnitLabel.TEST],
              cwd=chroot,
              synthetic_jar_dir=batch_output_dir,
              create_synthetic_jar=self.synthetic_classpath,
            )
            self.context.log.debug('JUnit subprocess exited with result ({})'
                                   .format(subprocess_result))
            result += abs(subprocess_result)

        tests_info = self.parse_test_info(batch_output_dir, parse_error_handler, ['classname'])
        for test_name, test_info in tests_info.items():
          test_item = Test(test_info['classname'], test_name)
          test_target = test_registry.get_owning_target(test_item)
          self.report_all_info_for_single_test(self.options_scope, test_target,
                                               test_name, test_info)

        if result != 0 and fail_fast:
          break

    if result == 0:
      return TestResult.rc(0)

    target_to_failed_test = parse_failed_targets(test_registry, output_dir, parse_error_handler)

    def sort_owning_target(t):
      return t.address.spec if t else None

    failed_targets = sorted(target_to_failed_test, key=sort_owning_target)
    error_message_lines = []
    if self._failure_summary:
      def render_owning_target(t):
        return t.address.reference() if t else '<Unknown Target>'

      for target in failed_targets:
        error_message_lines.append('\n{indent}{owner}'.format(indent=' ' * 4,
                                                              owner=render_owning_target(target)))
        for test in sorted(target_to_failed_test[target]):
          error_message_lines.append('{indent}{classname}#{methodname}'
                                     .format(indent=' ' * 8,
                                             classname=test.classname,
                                             methodname=test.methodname))
    error_message_lines.append(
      '\njava {main} ... exited non-zero ({code}); {failed} failed {targets}.'
        .format(main=JUnit.RUNNER_MAIN, code=result, failed=len(failed_targets),
                targets=pluralize(len(failed_targets), 'target'))
    )
    return TestResult(msg='\n'.join(error_message_lines), rc=result, failed_targets=failed_targets)
Exemplo n.º 13
0
 def platform(self):
     return JvmPlatform.global_instance().get_platform_for_target(self)
Exemplo n.º 14
0
  def console_output(self, targets):
    targets_map = {}
    resource_target_map = {}
    ivy_info = None
    if self.get_options().libraries:
      ivy_jar_products = self.context.products.get_data('ivy_jar_products') or {}
      # This product is a list for historical reasons (exclusives groups) but in practice should
      # have either 0 or 1 entries.
      ivy_info_list = ivy_jar_products.get('default')
      if ivy_info_list:
        assert len(ivy_info_list) == 1, (
          'The values in ivy_jar_products should always be length 1,'
          ' since we no longer have exclusives groups.'
        )
        ivy_info = ivy_info_list[0]

    ivy_jar_memo = {}

    def process_target(current_target):
      """
      :type current_target:pants.base.target.Target
      """
      def get_target_type(target):
        if target.is_test:
          return Export.SourceRootTypes.TEST
        else:
          if (isinstance(target, Resources) and
              target in resource_target_map and
              resource_target_map[target].is_test):
            return Export.SourceRootTypes.TEST_RESOURCE
          elif isinstance(target, Resources):
            return Export.SourceRootTypes.RESOURCE
          else:
            return Export.SourceRootTypes.SOURCE

      def get_transitive_jars(jar_lib):
        """
        :type jar_lib: pants.backend.jvm.targets.jar_library.JarLibrary
        :rtype: twitter.common.collections.orderedset.OrderedSet
        """
        if not ivy_info or not self.get_options().libraries:
          return OrderedSet()
        transitive_jars = OrderedSet()
        for jar in jar_lib.jar_dependencies:
          transitive_jars.update(ivy_info.get_jars_for_ivy_module(jar, memo=ivy_jar_memo))
        return transitive_jars

      info = {
        'targets': [],
        'libraries': [],
        'roots': [],
        'target_type': get_target_type(current_target),
        'is_code_gen': current_target.is_codegen,
        'pants_target_type': self._get_pants_target_alias(type(current_target))

      }

      if not current_target.is_synthetic:
        info['globs'] = current_target.globs_relative_to_buildroot()
        if self.get_options().sources:
          info['sources'] = list(current_target.sources_relative_to_buildroot())

      target_libraries = OrderedSet()
      if isinstance(current_target, JarLibrary):
        target_libraries = get_transitive_jars(current_target)
      for dep in current_target.dependencies:
        info['targets'].append(dep.address.spec)
        if isinstance(dep, JarLibrary):
          for jar in dep.jar_dependencies:
            target_libraries.add(IvyModuleRef(jar.org, jar.name, jar.rev))
          # Add all the jars pulled in by this jar_library
          target_libraries.update(get_transitive_jars(dep))
        if isinstance(dep, Resources):
          resource_target_map[dep] = current_target

      if isinstance(current_target, ScalaLibrary):
        for dep in current_target.java_sources:
          info['targets'].append(dep.address.spec)
          process_target(dep)

      if isinstance(current_target, JvmTarget):
        info['excludes'] = [self._exclude_id(exclude) for exclude in current_target.excludes]
        info['platform'] = current_target.platform.name

      info['roots'] = map(lambda (source_root, package_prefix): {
        'source_root': source_root,
        'package_prefix': package_prefix
      }, self._source_roots_for_target(current_target))

      if self.get_options().libraries:
        info['libraries'] = [self._jar_id(lib) for lib in target_libraries]
      targets_map[current_target.address.spec] = info

    for target in targets:
      process_target(target)

    jvm_platforms_map = {
      'default_platform' : JvmPlatform.global_instance().default_platform.name,
      'platforms': {
        str(platform_name): {
          'target_level' : str(platform.target_level),
          'source_level' : str(platform.source_level),
          'args' : platform.args,
        } for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items() }
    }

    graph_info = {
      'targets': targets_map,
      'jvm_platforms' : jvm_platforms_map,
    }
    jvm_distributions = DistributionLocator.global_instance().all_jdk_paths()
    if jvm_distributions:
      graph_info['jvm_distributions'] = jvm_distributions

    if self.get_options().libraries:
      graph_info['libraries'] = self._resolve_jars_info()

    graph_info['version'] = self.DEFAULT_EXPORT_VERSION

    if self.format:
      return json.dumps(graph_info, indent=4, separators=(',', ': ')).splitlines()
    else:
      return [json.dumps(graph_info)]
Exemplo n.º 15
0
    def console_output(self, targets):
        targets_map = {}
        resource_target_map = {}
        classpath_products = (
            self.context.products.get_data("compile_classpath") if self.get_options().libraries else None
        )

        python_interpreter_targets_mapping = defaultdict(list)

        def process_target(current_target):
            """
      :type current_target:pants.build_graph.target.Target
      """

            def get_target_type(target):
                if target.is_test:
                    return Export.SourceRootTypes.TEST
                else:
                    if (
                        isinstance(target, Resources)
                        and target in resource_target_map
                        and resource_target_map[target].is_test
                    ):
                        return Export.SourceRootTypes.TEST_RESOURCE
                    elif isinstance(target, Resources):
                        return Export.SourceRootTypes.RESOURCE
                    else:
                        return Export.SourceRootTypes.SOURCE

            info = {
                "targets": [],
                "libraries": [],
                "roots": [],
                "target_type": get_target_type(current_target),
                "is_code_gen": current_target.is_codegen,
                "pants_target_type": self._get_pants_target_alias(type(current_target)),
            }

            if not current_target.is_synthetic:
                info["globs"] = current_target.globs_relative_to_buildroot()
                if self.get_options().sources:
                    info["sources"] = list(current_target.sources_relative_to_buildroot())

            if isinstance(current_target, PythonRequirementLibrary):
                reqs = current_target.payload.get_field_value("requirements", set())
                """:type : set[pants.backend.python.python_requirement.PythonRequirement]"""
                info["requirements"] = [req.key for req in reqs]

            if isinstance(current_target, PythonTarget):
                interpreter_for_target = self.select_interpreter_for_targets([current_target])
                if interpreter_for_target is None:
                    raise TaskError("Unable to find suitable interpreter for {}".format(current_target.address))
                python_interpreter_targets_mapping[interpreter_for_target].append(current_target)
                info["python_interpreter"] = str(interpreter_for_target.identity)

            def iter_transitive_jars(jar_lib):
                """
        :type jar_lib: :class:`pants.backend.jvm.targets.jar_library.JarLibrary`
        :rtype: :class:`collections.Iterator` of
                :class:`pants.backend.jvm.jar_dependency_utils.M2Coordinate`
        """
                if classpath_products:
                    jar_products = classpath_products.get_artifact_classpath_entries_for_targets((jar_lib,))
                    for _, jar_entry in jar_products:
                        coordinate = jar_entry.coordinate
                        # We drop classifier and type_ since those fields are represented in the global
                        # libraries dict and here we just want the key into that dict (see `_jar_id`).
                        yield M2Coordinate(org=coordinate.org, name=coordinate.name, rev=coordinate.rev)

            target_libraries = OrderedSet()
            if isinstance(current_target, JarLibrary):
                target_libraries = OrderedSet(iter_transitive_jars(current_target))
            for dep in current_target.dependencies:
                info["targets"].append(dep.address.spec)
                if isinstance(dep, JarLibrary):
                    for jar in dep.jar_dependencies:
                        target_libraries.add(M2Coordinate(jar.org, jar.name, jar.rev))
                    # Add all the jars pulled in by this jar_library
                    target_libraries.update(iter_transitive_jars(dep))
                if isinstance(dep, Resources):
                    resource_target_map[dep] = current_target

            if isinstance(current_target, ScalaLibrary):
                for dep in current_target.java_sources:
                    info["targets"].append(dep.address.spec)
                    process_target(dep)

            if isinstance(current_target, JvmTarget):
                info["excludes"] = [self._exclude_id(exclude) for exclude in current_target.excludes]
                info["platform"] = current_target.platform.name

            info["roots"] = map(
                lambda (source_root, package_prefix): {"source_root": source_root, "package_prefix": package_prefix},
                self._source_roots_for_target(current_target),
            )

            if classpath_products:
                info["libraries"] = [self._jar_id(lib) for lib in target_libraries]
            targets_map[current_target.address.spec] = info

        for target in targets:
            process_target(target)

        jvm_platforms_map = {
            "default_platform": JvmPlatform.global_instance().default_platform.name,
            "platforms": {
                str(platform_name): {
                    "target_level": str(platform.target_level),
                    "source_level": str(platform.source_level),
                    "args": platform.args,
                }
                for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items()
            },
        }

        graph_info = {
            "version": self.DEFAULT_EXPORT_VERSION,
            "targets": targets_map,
            "jvm_platforms": jvm_platforms_map,
        }
        jvm_distributions = DistributionLocator.global_instance().all_jdk_paths()
        if jvm_distributions:
            graph_info["jvm_distributions"] = jvm_distributions

        if classpath_products:
            graph_info["libraries"] = self._resolve_jars_info(targets, classpath_products)

        if python_interpreter_targets_mapping:
            interpreters = self.interpreter_cache.select_interpreter(python_interpreter_targets_mapping.keys())
            default_interpreter = interpreters[0]

            interpreters_info = {}
            for interpreter, targets in six.iteritems(python_interpreter_targets_mapping):
                chroot = self.cached_chroot(interpreter=interpreter, pex_info=PexInfo.default(), targets=targets)
                interpreters_info[str(interpreter.identity)] = {"binary": interpreter.binary, "chroot": chroot.path()}

            graph_info["python_setup"] = {
                "default_interpreter": str(default_interpreter.identity),
                "interpreters": interpreters_info,
            }

        if self.format:
            return json.dumps(graph_info, indent=4, separators=(",", ": ")).splitlines()
        else:
            return [json.dumps(graph_info)]
Exemplo n.º 16
0
    def work_for_vts_rsc_jar_library(vts, ctx):

      cp_entries = []

      # Include the current machine's jdk lib jars. This'll blow up remotely.
      # We need a solution for that.
      # Probably something to do with https://github.com/pantsbuild/pants/pull/6346
      # TODO perhaps determine the platform of the jar and use that here.
      # https://github.com/pantsbuild/pants/issues/6547
      distribution = JvmPlatform.preferred_jvm_distribution([], strict=True)
      jvm_lib_jars_abs = distribution.find_libs(['rt.jar', 'dt.jar', 'jce.jar', 'tools.jar'])
      cp_entries.extend(jvm_lib_jars_abs)

      # TODO use compile_classpath
      classpath_abs = [
        path for (conf, path) in
        self.context.products.get_data('rsc_classpath').get_for_target(ctx.target)
      ]
      dependency_classpath = self._zinc.compile_classpath(
        'compile_classpath',
        ctx.target,
        extra_cp_entries=self._extra_compile_time_classpath)
      classpath_rel = fast_relpath_collection(classpath_abs)
      cp_entries.extend(classpath_rel)

      counter_val = str(counter()).rjust(counter.format_length(), b' ')
      counter_str = '[{}/{}] '.format(counter_val, counter.size)
      self.context.log.info(
        counter_str,
        'Metacp-ing ',
        items_to_report_element(cp_entries, 'jar'),
        ' in ',
        items_to_report_element([t.address.reference() for t in vts.targets], 'target'),
        ' (',
        ctx.target.address.spec,
        ').')

      ctx.ensure_output_dirs_exist()

      tgt, = vts.targets
      with Timer() as timer:
      # Step 1: Convert classpath to SemanticDB
        # ---------------------------------------
        scalac_classpath_path_entries_abs = self.tool_classpath('workaround-metacp-dependency-classpath')
        scalac_classpath_path_entries = fast_relpath_collection(scalac_classpath_path_entries_abs)
        rsc_index_dir = fast_relpath(ctx.rsc_index_dir, get_buildroot())
        args = [
          '--verbose',
          # NB: Without this setting, rsc will be missing some symbols
          #     from the scala library.
          '--include-scala-library-synthetics', # TODO generate these once and cache them
          # NB: We need to add these extra dependencies in order to be able
          #     to find symbols used by the scalac jars.
          '--dependency-classpath', os.pathsep.join(dependency_classpath + scalac_classpath_path_entries),
          # NB: The directory to dump the semanticdb jars generated by metacp.
          '--out', rsc_index_dir,
          os.pathsep.join(cp_entries),
        ]
        metacp_wu = self._runtool(
          'scala.meta.cli.Metacp',
          'metacp',
          args,
          distribution,
          tgt=tgt,
          input_files=(scalac_classpath_path_entries + classpath_rel),
          output_dir=rsc_index_dir)
        metacp_stdout = stdout_contents(metacp_wu)
        metacp_result = json.loads(metacp_stdout)

        metai_classpath = self._collect_metai_classpath(
          metacp_result, classpath_rel, jvm_lib_jars_abs)

        # Step 1.5: metai Index the semanticdbs
        # -------------------------------------
        self._run_metai_tool(distribution, metai_classpath, rsc_index_dir, tgt)

        abs_output = [(conf, os.path.join(get_buildroot(), x))
                      for conf in self._confs for x in metai_classpath]

        self._metacp_jars_classpath_product.add_for_target(
          ctx.target,
          abs_output,
        )

      self._record_target_stats(tgt,
          len(abs_output),
          len([]),
          timer.elapsed,
          False,
          'metacp'
        )
Exemplo n.º 17
0
    def generate_targets_map(self, targets, classpath_products=None):
        """Generates a dictionary containing all pertinent information about the target graph.

    The return dictionary is suitable for serialization by json.dumps.
    :param targets: The list of targets to generate the map for.
    :param classpath_products: Optional classpath_products. If not provided when the --libraries
      option is `True`, this task will perform its own jar resolution.
    """
        targets_map = {}
        resource_target_map = {}
        python_interpreter_targets_mapping = defaultdict(list)

        if self.get_options().libraries:
            # NB(gmalmquist): This supports mocking the classpath_products in tests.
            if classpath_products is None:
                classpath_products = self.resolve_jars(targets)
        else:
            classpath_products = None

        target_roots_set = set(self.context.target_roots)

        def process_target(current_target):
            """
      :type current_target:pants.build_graph.target.Target
      """
            def get_target_type(tgt):
                def is_test(t):
                    return isinstance(t, JUnitTests) or isinstance(
                        t, PythonTests)

                if is_test(tgt):
                    return SourceRootTypes.TEST
                else:
                    if (isinstance(tgt, Resources)
                            and tgt in resource_target_map
                            and is_test(resource_target_map[tgt])):
                        return SourceRootTypes.TEST_RESOURCE
                    elif isinstance(tgt, Resources):
                        return SourceRootTypes.RESOURCE
                    else:
                        return SourceRootTypes.SOURCE

            info = {
                'targets': [],
                'libraries': [],
                'roots': [],
                'id':
                current_target.id,
                'target_type':
                get_target_type(current_target),
                # NB: is_code_gen should be removed when export format advances to 1.1.0 or higher
                'is_code_gen':
                current_target.is_synthetic,
                'is_synthetic':
                current_target.is_synthetic,
                'pants_target_type':
                self._get_pants_target_alias(type(current_target)),
            }

            if not current_target.is_synthetic:
                info['globs'] = current_target.globs_relative_to_buildroot()
                if self.get_options().sources:
                    info['sources'] = list(
                        current_target.sources_relative_to_buildroot())

            info['transitive'] = current_target.transitive
            info['scope'] = str(current_target.scope)
            info['is_target_root'] = current_target in target_roots_set

            if isinstance(current_target, PythonRequirementLibrary):
                reqs = current_target.payload.get_field_value(
                    'requirements', set())
                """:type : set[pants.backend.python.python_requirement.PythonRequirement]"""
                info['requirements'] = [req.key for req in reqs]

            if isinstance(current_target, PythonTarget):
                interpreter_for_target = self._interpreter_cache.select_interpreter_for_targets(
                    [current_target])
                if interpreter_for_target is None:
                    raise TaskError(
                        'Unable to find suitable interpreter for {}'.format(
                            current_target.address))
                python_interpreter_targets_mapping[
                    interpreter_for_target].append(current_target)
                info['python_interpreter'] = str(
                    interpreter_for_target.identity)

            def iter_transitive_jars(jar_lib):
                """
        :type jar_lib: :class:`pants.backend.jvm.targets.jar_library.JarLibrary`
        :rtype: :class:`collections.Iterator` of
                :class:`pants.java.jar.M2Coordinate`
        """
                if classpath_products:
                    jar_products = classpath_products.get_artifact_classpath_entries_for_targets(
                        (jar_lib, ))
                    for _, jar_entry in jar_products:
                        coordinate = jar_entry.coordinate
                        # We drop classifier and type_ since those fields are represented in the global
                        # libraries dict and here we just want the key into that dict (see `_jar_id`).
                        yield M2Coordinate(org=coordinate.org,
                                           name=coordinate.name,
                                           rev=coordinate.rev)

            target_libraries = OrderedSet()
            if isinstance(current_target, JarLibrary):
                target_libraries = OrderedSet(
                    iter_transitive_jars(current_target))
            for dep in current_target.dependencies:
                info['targets'].append(dep.address.spec)
                if isinstance(dep, JarLibrary):
                    for jar in dep.jar_dependencies:
                        target_libraries.add(
                            M2Coordinate(jar.org, jar.name, jar.rev))
                    # Add all the jars pulled in by this jar_library
                    target_libraries.update(iter_transitive_jars(dep))
                if isinstance(dep, Resources):
                    resource_target_map[dep] = current_target

            if isinstance(current_target, ScalaLibrary):
                for dep in current_target.java_sources:
                    info['targets'].append(dep.address.spec)
                    process_target(dep)

            if isinstance(current_target, JvmTarget):
                info['excludes'] = [
                    self._exclude_id(exclude)
                    for exclude in current_target.excludes
                ]
                info['platform'] = current_target.platform.name
                if hasattr(current_target, 'test_platform'):
                    info['test_platform'] = current_target.test_platform.name

            info['roots'] = [{
                'source_root': source_root_package_prefix[0],
                'package_prefix': source_root_package_prefix[1]
            } for source_root_package_prefix in self._source_roots_for_target(
                current_target)]

            if classpath_products:
                info['libraries'] = [
                    self._jar_id(lib) for lib in target_libraries
                ]
            targets_map[current_target.address.spec] = info

        for target in targets:
            process_target(target)

        scala_platform = ScalaPlatform.global_instance()
        scala_platform_map = {
            'scala_version':
            scala_platform.version,
            'compiler_classpath': [
                cp_entry.path
                for cp_entry in scala_platform.compiler_classpath_entries(
                    self.context.products)
            ],
        }

        jvm_platforms_map = {
            'default_platform':
            JvmPlatform.global_instance().default_platform.name,
            'platforms': {
                str(platform_name): {
                    'target_level': str(platform.target_level),
                    'source_level': str(platform.source_level),
                    'args': platform.args,
                }
                for platform_name, platform in
                JvmPlatform.global_instance().platforms_by_name.items()
            },
        }

        graph_info = {
            'version': DEFAULT_EXPORT_VERSION,
            'targets': targets_map,
            'jvm_platforms': jvm_platforms_map,
            'scala_platform': scala_platform_map,
            # `jvm_distributions` are static distribution settings from config,
            # `preferred_jvm_distributions` are distributions that pants actually uses for the
            # given platform setting.
            'preferred_jvm_distributions': {}
        }

        for platform_name, platform in JvmPlatform.global_instance(
        ).platforms_by_name.items():
            preferred_distributions = {}
            for strict, strict_key in [(True, 'strict'),
                                       (False, 'non_strict')]:
                try:
                    dist = JvmPlatform.preferred_jvm_distribution(
                        [platform], strict=strict)
                    preferred_distributions[strict_key] = dist.home
                except DistributionLocator.Error:
                    pass

            if preferred_distributions:
                graph_info['preferred_jvm_distributions'][
                    platform_name] = preferred_distributions

        if classpath_products:
            graph_info['libraries'] = self._resolve_jars_info(
                targets, classpath_products)

        if python_interpreter_targets_mapping:
            # NB: We've selected a python interpreter compatible with each python target individually into
            # the `python_interpreter_targets_mapping`. These python targets may not be compatible, ie: we
            # could have a python target requiring 'CPython>=2.7<3' (ie: CPython-2.7.x) and another
            # requiring 'CPython>=3.6'. To pick a default interpreter then from among these two choices
            # is arbitrary and not to be relied on to work as a default interpreter if ever needed by the
            # export consumer.
            #
            # TODO(John Sirois): consider either eliminating the 'default_interpreter' field and pressing
            # export consumers to make their own choice of a default (if needed) or else use
            # `select.select_interpreter_for_targets` and fail fast if there is no interpreter compatible
            # across all the python targets in-play.
            #
            # For now, make our arbitrary historical choice of a default interpreter explicit and use the
            # lowest version.
            default_interpreter = min(
                python_interpreter_targets_mapping.keys())

            interpreters_info = {}
            for interpreter, targets in python_interpreter_targets_mapping.items(
            ):
                req_libs = [
                    target for target in Target.closure_for_targets(targets)
                    if has_python_requirements(target)
                ]
                chroot = self.resolve_requirements(interpreter, req_libs)
                interpreters_info[str(interpreter.identity)] = {
                    'binary': interpreter.binary,
                    'chroot': chroot.path()
                }

            graph_info['python_setup'] = {
                'default_interpreter': str(default_interpreter.identity),
                'interpreters': interpreters_info
            }

        if self.get_options().available_target_types:
            graph_info['available_target_types'] = self._target_types()

        return graph_info
Exemplo n.º 18
0
  def _run_tests(self, test_registry, output_dir, coverage=None):
    if coverage:
      extra_jvm_options = coverage.extra_jvm_options
      classpath_prepend = coverage.classpath_prepend
      classpath_append = coverage.classpath_append
    else:
      extra_jvm_options = []
      classpath_prepend = ()
      classpath_append = ()

    tests_by_properties = test_registry.index(
        lambda tgt: tgt.cwd if tgt.cwd is not None else self._working_dir,
        lambda tgt: tgt.test_platform,
        lambda tgt: tgt.payload.extra_jvm_options,
        lambda tgt: tgt.payload.extra_env_vars,
        lambda tgt: tgt.concurrency,
        lambda tgt: tgt.threads)

    # the below will be None if not set, and we'll default back to runtime_classpath
    classpath_product = self.context.products.get_data('instrument_classpath')

    result = 0
    for properties, tests in tests_by_properties.items():
      (workdir, platform, target_jvm_options, target_env_vars, concurrency, threads) = properties
      for batch in self._partition(tests):
        # Batches of test classes will likely exist within the same targets: dedupe them.
        relevant_targets = {test_registry.get_owning_target(t) for t in batch}
        complete_classpath = OrderedSet()
        complete_classpath.update(classpath_prepend)
        complete_classpath.update(JUnit.global_instance().runner_classpath(self.context))
        complete_classpath.update(self.classpath(relevant_targets,
                                                 classpath_product=classpath_product))
        complete_classpath.update(classpath_append)
        distribution = JvmPlatform.preferred_jvm_distribution([platform], self._strict_jvm_version)

        # Override cmdline args with values from junit_test() target that specify concurrency:
        args = self._args(output_dir) + [u'-xmlreport']

        if concurrency is not None:
          args = remove_arg(args, '-default-parallel')
          if concurrency == JUnitTests.CONCURRENCY_SERIAL:
            args = ensure_arg(args, '-default-concurrency', param='SERIAL')
          elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES:
            args = ensure_arg(args, '-default-concurrency', param='PARALLEL_CLASSES')
          elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_METHODS:
            args = ensure_arg(args, '-default-concurrency', param='PARALLEL_METHODS')
          elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES_AND_METHODS:
            args = ensure_arg(args, '-default-concurrency', param='PARALLEL_CLASSES_AND_METHODS')

        if threads is not None:
          args = remove_arg(args, '-parallel-threads', has_param=True)
          args += ['-parallel-threads', str(threads)]

        batch_test_specs = [test.render_test_spec() for test in batch]
        with argfile.safe_args(batch_test_specs, self.get_options()) as batch_tests:
          self.context.log.debug('CWD = {}'.format(workdir))
          self.context.log.debug('platform = {}'.format(platform))
          with environment_as(**dict(target_env_vars)):
            result += abs(self._spawn_and_wait(
              executor=SubprocessExecutor(distribution),
              distribution=distribution,
              classpath=complete_classpath,
              main=JUnit.RUNNER_MAIN,
              jvm_options=self.jvm_options + extra_jvm_options + list(target_jvm_options),
              args=args + batch_tests,
              workunit_factory=self.context.new_workunit,
              workunit_name='run',
              workunit_labels=[WorkUnitLabel.TEST],
              cwd=workdir,
              synthetic_jar_dir=output_dir,
              create_synthetic_jar=self.synthetic_classpath,
            ))

          if result != 0 and self._fail_fast:
            break

    if result != 0:
      def error_handler(parse_error):
        # Just log and move on since the result is only used to characterize failures, and raising
        # an error here would just distract from the underlying test failures.
        self.context.log.error('Error parsing test result file {path}: {cause}'
                               .format(path=parse_error.junit_xml_path, cause=parse_error.cause))

      target_to_failed_test = parse_failed_targets(test_registry, output_dir, error_handler)
      failed_targets = sorted(target_to_failed_test, key=lambda t: t.address.spec)
      error_message_lines = []
      if self._failure_summary:
        for target in failed_targets:
          error_message_lines.append('\n{indent}{address}'.format(indent=' ' * 4,
                                                                  address=target.address.spec))
          for test in sorted(target_to_failed_test[target]):
            error_message_lines.append('{indent}{classname}#{methodname}'
                                       .format(indent=' ' * 8,
                                               classname=test.classname,
                                               methodname=test.methodname))
      error_message_lines.append(
        '\njava {main} ... exited non-zero ({code}); {failed} failed {targets}.'
          .format(main=JUnit.RUNNER_MAIN, code=result, failed=len(failed_targets),
                  targets=pluralize(len(failed_targets), 'target'))
      )
      raise TestFailedTaskError('\n'.join(error_message_lines), failed_targets=list(failed_targets))
Exemplo n.º 19
0
    def compile(self, ctx, args, classpath, upstream_analysis, settings,
                fatal_warnings, zinc_file_manager, javac_plugin_map,
                scalac_plugin_map):
        try:
            distribution = JvmPlatform.preferred_jvm_distribution([settings],
                                                                  strict=True)
        except DistributionLocator.Error:
            distribution = JvmPlatform.preferred_jvm_distribution([settings],
                                                                  strict=False)

        javac_cmd = ['{}/bin/javac'.format(distribution.real_home)]

        javac_cmd.extend([
            '-classpath',
            ':'.join(classpath),
        ])

        if settings.args:
            settings_args = settings.args
            if any('$JAVA_HOME' in a for a in settings.args):
                logger.debug(
                    'Substituting "$JAVA_HOME" with "{}" in jvm-platform args.'
                    .format(distribution.home))
                settings_args = (a.replace('$JAVA_HOME', distribution.home)
                                 for a in settings.args)
            javac_cmd.extend(settings_args)

        javac_cmd.extend([
            '-d',
            ctx.classes_dir,
            # TODO: support -release
            '-source',
            str(settings.source_level),
            '-target',
            str(settings.target_level),
        ])

        javac_cmd.extend(self._javac_plugin_args(javac_plugin_map))

        javac_cmd.extend(args)

        if fatal_warnings:
            javac_cmd.extend(self.get_options().fatal_warnings_enabled_args)
        else:
            javac_cmd.extend(self.get_options().fatal_warnings_disabled_args)

        with argfile.safe_args(ctx.sources,
                               self.get_options()) as batched_sources:
            javac_cmd.extend(batched_sources)

            with self.context.new_workunit(name='javac',
                                           cmd=' '.join(javac_cmd),
                                           labels=[WorkUnitLabel.COMPILER
                                                   ]) as workunit:
                self.context.log.debug('Executing {}'.format(
                    ' '.join(javac_cmd)))
                p = subprocess.Popen(javac_cmd,
                                     stdout=workunit.output('stdout'),
                                     stderr=workunit.output('stderr'))
                return_code = p.wait()
                workunit.set_outcome(
                    WorkUnit.FAILURE if return_code else WorkUnit.SUCCESS)
                if return_code:
                    raise TaskError(
                        'javac exited with return code {rc}'.format(
                            rc=return_code))
Exemplo n.º 20
0
  def _run_tests(self, tests_to_targets):
    if self._coverage:
      extra_jvm_options = self._coverage.extra_jvm_options
      classpath_prepend = self._coverage.classpath_prepend
      classpath_append = self._coverage.classpath_append
    else:
      extra_jvm_options = []
      classpath_prepend = ()
      classpath_append = ()

    tests_by_properties = self._tests_by_properties(
      tests_to_targets,
      self._infer_workdir,
      lambda target: target.test_platform,
      lambda target: target.payload.extra_jvm_options,
      lambda target: target.payload.extra_env_vars,
    )

    # the below will be None if not set, and we'll default back to runtime_classpath
    classpath_product = self.context.products.get_data('instrument_classpath')

    result = 0
    for (workdir, platform, target_jvm_options, target_env_vars), tests in tests_by_properties.items():
      for batch in self._partition(tests):
        # Batches of test classes will likely exist within the same targets: dedupe them.
        relevant_targets = set(map(tests_to_targets.get, batch))
        complete_classpath = OrderedSet()
        complete_classpath.update(classpath_prepend)
        complete_classpath.update(self.tool_classpath('junit'))
        complete_classpath.update(self.classpath(relevant_targets,
                                                 classpath_product=classpath_product))
        complete_classpath.update(classpath_append)
        distribution = JvmPlatform.preferred_jvm_distribution([platform], self._strict_jvm_version)
        with binary_util.safe_args(batch, self.get_options()) as batch_tests:
          self.context.log.debug('CWD = {}'.format(workdir))
          self.context.log.debug('platform = {}'.format(platform))
          with environment_as(**dict(target_env_vars)):
            result += abs(self._spawn_and_wait(
              executor=SubprocessExecutor(distribution),
              distribution=distribution,
              classpath=complete_classpath,
              main=JUnitRun._MAIN,
              jvm_options=self.jvm_options + extra_jvm_options + list(target_jvm_options),
              args=self._args + batch_tests + [u'-xmlreport'],
              workunit_factory=self.context.new_workunit,
              workunit_name='run',
              workunit_labels=[WorkUnitLabel.TEST],
              cwd=workdir,
              synthetic_jar_dir=self.workdir,
              create_synthetic_jar=self.synthetic_classpath,
            ))

          if result != 0 and self._fail_fast:
            break

    if result != 0:
      failed_targets_and_tests = self._get_failed_targets(tests_to_targets)
      failed_targets = sorted(failed_targets_and_tests, key=lambda target: target.address.spec)
      error_message_lines = []
      if self._failure_summary:
        for target in failed_targets:
          error_message_lines.append('\n{0}{1}'.format(' '*4, target.address.spec))
          for test in sorted(failed_targets_and_tests[target]):
            error_message_lines.append('{0}{1}'.format(' '*8, test))
      error_message_lines.append(
        '\njava {main} ... exited non-zero ({code}); {failed} failed {targets}.'
          .format(main=JUnitRun._MAIN, code=result, failed=len(failed_targets),
                  targets=pluralize(len(failed_targets), 'target'))
      )
      raise TestFailedTaskError('\n'.join(error_message_lines), failed_targets=list(failed_targets))
Exemplo n.º 21
0
  def generate_targets_map(self, targets, classpath_products=None):
    """Generates a dictionary containing all pertinent information about the target graph.

    The return dictionary is suitable for serialization by json.dumps.
    :param targets: The list of targets to generate the map for.
    :param classpath_products: Optional classpath_products. If not provided when the --libraries
      option is `True`, this task will perform its own jar resolution.
    """
    targets_map = {}
    resource_target_map = {}
    python_interpreter_targets_mapping = defaultdict(list)

    if self.get_options().libraries:
      # NB(gmalmquist): This supports mocking the classpath_products in tests.
      if classpath_products is None:
        classpath_products = self.resolve_jars(targets)
    else:
      classpath_products = None

    target_roots_set = set(self.context.target_roots)

    def process_target(current_target):
      """
      :type current_target:pants.build_graph.target.Target
      """
      def get_target_type(tgt):
        def is_test(t):
          return isinstance(t, JUnitTests) or isinstance(t, PythonTests)
        if is_test(tgt):
          return ExportTask.SourceRootTypes.TEST
        else:
          if (isinstance(tgt, Resources) and
              tgt in resource_target_map and
                is_test(resource_target_map[tgt])):
            return ExportTask.SourceRootTypes.TEST_RESOURCE
          elif isinstance(tgt, Resources):
            return ExportTask.SourceRootTypes.RESOURCE
          else:
            return ExportTask.SourceRootTypes.SOURCE

      info = {
        'targets': [],
        'libraries': [],
        'roots': [],
        'id': current_target.id,
        'target_type': get_target_type(current_target),
        # NB: is_code_gen should be removed when export format advances to 1.1.0 or higher
        'is_code_gen': current_target.is_synthetic,
        'is_synthetic': current_target.is_synthetic,
        'pants_target_type': self._get_pants_target_alias(type(current_target)),
      }

      if not current_target.is_synthetic:
        info['globs'] = current_target.globs_relative_to_buildroot()
        if self.get_options().sources:
          info['sources'] = list(current_target.sources_relative_to_buildroot())

      info['transitive'] = current_target.transitive
      info['scope'] = str(current_target.scope)
      info['is_target_root'] = current_target in target_roots_set

      if isinstance(current_target, PythonRequirementLibrary):
        reqs = current_target.payload.get_field_value('requirements', set())
        """:type : set[pants.backend.python.python_requirement.PythonRequirement]"""
        info['requirements'] = [req.key for req in reqs]

      if isinstance(current_target, PythonTarget):
        interpreter_for_target = self._interpreter_cache.select_interpreter_for_targets(
          [current_target])
        if interpreter_for_target is None:
          raise TaskError('Unable to find suitable interpreter for {}'
                          .format(current_target.address))
        python_interpreter_targets_mapping[interpreter_for_target].append(current_target)
        info['python_interpreter'] = str(interpreter_for_target.identity)

      def iter_transitive_jars(jar_lib):
        """
        :type jar_lib: :class:`pants.backend.jvm.targets.jar_library.JarLibrary`
        :rtype: :class:`collections.Iterator` of
                :class:`pants.java.jar.M2Coordinate`
        """
        if classpath_products:
          jar_products = classpath_products.get_artifact_classpath_entries_for_targets((jar_lib,))
          for _, jar_entry in jar_products:
            coordinate = jar_entry.coordinate
            # We drop classifier and type_ since those fields are represented in the global
            # libraries dict and here we just want the key into that dict (see `_jar_id`).
            yield M2Coordinate(org=coordinate.org, name=coordinate.name, rev=coordinate.rev)

      target_libraries = OrderedSet()
      if isinstance(current_target, JarLibrary):
        target_libraries = OrderedSet(iter_transitive_jars(current_target))
      for dep in current_target.dependencies:
        info['targets'].append(dep.address.spec)
        if isinstance(dep, JarLibrary):
          for jar in dep.jar_dependencies:
            target_libraries.add(M2Coordinate(jar.org, jar.name, jar.rev))
          # Add all the jars pulled in by this jar_library
          target_libraries.update(iter_transitive_jars(dep))
        if isinstance(dep, Resources):
          resource_target_map[dep] = current_target

      if isinstance(current_target, ScalaLibrary):
        for dep in current_target.java_sources:
          info['targets'].append(dep.address.spec)
          process_target(dep)

      if isinstance(current_target, JvmTarget):
        info['excludes'] = [self._exclude_id(exclude) for exclude in current_target.excludes]
        info['platform'] = current_target.platform.name
        if hasattr(current_target, 'test_platform'):
          info['test_platform'] = current_target.test_platform.name

      info['roots'] = [{
        'source_root': source_root_package_prefix[0],
        'package_prefix': source_root_package_prefix[1]
      } for source_root_package_prefix in self._source_roots_for_target(current_target)]

      if classpath_products:
        info['libraries'] = [self._jar_id(lib) for lib in target_libraries]
      targets_map[current_target.address.spec] = info

    for target in targets:
      process_target(target)

    jvm_platforms_map = {
      'default_platform' : JvmPlatform.global_instance().default_platform.name,
      'platforms': {
        str(platform_name): {
          'target_level' : str(platform.target_level),
          'source_level' : str(platform.source_level),
          'args' : platform.args,
        } for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items() },
    }

    graph_info = {
      'version': self.DEFAULT_EXPORT_VERSION,
      'targets': targets_map,
      'jvm_platforms': jvm_platforms_map,
      # `jvm_distributions` are static distribution settings from config,
      # `preferred_jvm_distributions` are distributions that pants actually uses for the
      # given platform setting.
      'preferred_jvm_distributions': {}
    }

    for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items():
      preferred_distributions = {}
      for strict, strict_key in [(True, 'strict'), (False, 'non_strict')]:
        try:
          dist = JvmPlatform.preferred_jvm_distribution([platform], strict=strict)
          preferred_distributions[strict_key] = dist.home
        except DistributionLocator.Error:
          pass

      if preferred_distributions:
        graph_info['preferred_jvm_distributions'][platform_name] = preferred_distributions

    if classpath_products:
      graph_info['libraries'] = self._resolve_jars_info(targets, classpath_products)

    if python_interpreter_targets_mapping:
      # NB: We've selected a python interpreter compatible with each python target individually into
      # the `python_interpreter_targets_mapping`. These python targets may not be compatible, ie: we
      # could have a python target requiring 'CPython>=2.7<3' (ie: CPython-2.7.x) and another
      # requiring 'CPython>=3.6'. To pick a default interpreter then from among these two choices
      # is arbitrary and not to be relied on to work as a default interpreter if ever needed by the
      # export consumer.
      #
      # TODO(John Sirois): consider either eliminating the 'default_interpreter' field and pressing
      # export consumers to make their own choice of a default (if needed) or else use
      # `select.select_interpreter_for_targets` and fail fast if there is no interpreter compatible
      # across all the python targets in-play.
      #
      # For now, make our arbitrary historical choice of a default interpreter explicit and use the
      # lowest version.
      default_interpreter = min(python_interpreter_targets_mapping.keys())

      interpreters_info = {}
      for interpreter, targets in six.iteritems(python_interpreter_targets_mapping):
        req_libs = [target for target in Target.closure_for_targets(targets)
                    if has_python_requirements(target)]
        chroot = self.resolve_requirements(interpreter, req_libs)
        interpreters_info[str(interpreter.identity)] = {
          'binary': interpreter.binary,
          'chroot': chroot.path()
        }

      graph_info['python_setup'] = {
        'default_interpreter': str(default_interpreter.identity),
        'interpreters': interpreters_info
      }

    return graph_info
Exemplo n.º 22
0
    def compile(
        self,
        ctx,
        args,
        dependency_classpath,
        upstream_analysis,
        settings,
        compiler_option_sets,
        zinc_file_manager,
        javac_plugin_map,
        scalac_plugin_map,
    ):
        classpath = (ctx.classes_dir.path, ) + tuple(
            ce.path for ce in dependency_classpath)

        if self.get_options().capture_classpath:
            self._record_compile_classpath(classpath, ctx.target,
                                           ctx.classes_dir.path)

        try:
            distribution = JvmPlatform.preferred_jvm_distribution([settings],
                                                                  strict=True)
        except DistributionLocator.Error:
            distribution = JvmPlatform.preferred_jvm_distribution([settings],
                                                                  strict=False)

        javac_args = []

        if settings.args:
            settings_args = settings.args
            if any("$JAVA_HOME" in a for a in settings.args):
                logger.debug(
                    'Substituting "$JAVA_HOME" with "{}" in jvm-platform args.'
                    .format(distribution.home))
                settings_args = (a.replace("$JAVA_HOME", distribution.home)
                                 for a in settings.args)
            javac_args.extend(settings_args)

            javac_args.extend([
                # TODO: support -release
                "-source",
                str(settings.source_level),
                "-target",
                str(settings.target_level),
            ])

        if self.execution_strategy == self.ExecutionStrategy.hermetic:
            javac_args.extend([
                # We need to strip the source root from our output files. Outputting to a directory, and
                # capturing that directory, does the job.
                # Unfortunately, javac errors if the directory you pass to -d doesn't exist, and we don't
                # have a convenient way of making a directory in the output tree, so let's just use the
                # working directory as our output dir.
                # This also has the benefit of not needing to strip leading directories from the returned
                # snapshot.
                "-d",
                ".",
            ])
        else:
            javac_args.extend(["-d", ctx.classes_dir.path])

        javac_args.extend(self._javac_plugin_args(javac_plugin_map))

        javac_args.extend(args)

        compiler_option_sets_args = self.get_merged_args_for_compiler_option_sets(
            compiler_option_sets)
        javac_args.extend(compiler_option_sets_args)

        javac_args.extend(["-classpath", ":".join(classpath)])
        javac_args.extend(ctx.sources)

        # From https://docs.oracle.com/javase/8/docs/technotes/tools/windows/javac.html#BHCJEIBB
        # Wildcards (*) aren’t allowed in these lists (such as for specifying *.java).
        # Use of the at sign (@) to recursively interpret files isn’t supported.
        # The -J options aren’t supported because they’re passed to the launcher,
        # which doesn’t support argument files.
        j_args = [j_arg for j_arg in javac_args if j_arg.startswith("-J")]
        safe_javac_args = list(filter(lambda x: x not in j_args, javac_args))

        with argfile.safe_args(safe_javac_args,
                               self.get_options()) as batched_args:
            javac_cmd = [f"{distribution.real_home}/bin/javac"]
            javac_cmd.extend(j_args)
            javac_cmd.extend(batched_args)

            if self.execution_strategy == self.ExecutionStrategy.hermetic:
                self._execute_hermetic_compile(javac_cmd, ctx)
            else:
                with self.context.new_workunit(name="javac",
                                               cmd=" ".join(javac_cmd),
                                               labels=[WorkUnitLabel.COMPILER
                                                       ]) as workunit:
                    self.context.log.debug(f"Executing {' '.join(javac_cmd)}")
                    p = subprocess.Popen(
                        javac_cmd,
                        stdout=workunit.output("stdout"),
                        stderr=workunit.output("stderr"),
                    )
                    return_code = p.wait()
                    workunit.set_outcome(
                        WorkUnit.FAILURE if return_code else WorkUnit.SUCCESS)
                    if return_code:
                        raise TaskError(
                            f"javac exited with return code {return_code}")
                classes_directory = Path(ctx.classes_dir.path).relative_to(
                    get_buildroot())
                self.context._scheduler.materialize_directory(
                    DirectoryToMaterialize(
                        self.post_compile_extra_resources_digest(
                            ctx, prepend_post_merge_relative_path=False),
                        path_prefix=str(classes_directory),
                    ), )

        self._create_context_jar(ctx)
Exemplo n.º 23
0
  def compile(self, ctx, args, classpath, upstream_analysis,
              settings, fatal_warnings, zinc_file_manager,
              javac_plugin_map, scalac_plugin_map):
    try:
      distribution = JvmPlatform.preferred_jvm_distribution([settings], strict=True)
    except DistributionLocator.Error:
      distribution = JvmPlatform.preferred_jvm_distribution([settings], strict=False)

    javac_cmd = ['{}/bin/javac'.format(distribution.real_home)]

    javac_cmd.extend([
      '-classpath', ':'.join(classpath),
    ])

    if settings.args:
      settings_args = settings.args
      if any('$JAVA_HOME' in a for a in settings.args):
        logger.debug('Substituting "$JAVA_HOME" with "{}" in jvm-platform args.'
                     .format(distribution.home))
        settings_args = (a.replace('$JAVA_HOME', distribution.home) for a in settings.args)
      javac_cmd.extend(settings_args)

      javac_cmd.extend([
        # TODO: support -release
        '-source', str(settings.source_level),
        '-target', str(settings.target_level),
      ])

    if self.execution_strategy == self.HERMETIC:
      javac_cmd.extend([
        # We need to strip the source root from our output files. Outputting to a directory, and
        # capturing that directory, does the job.
        # Unfortunately, javac errors if the directory you pass to -d doesn't exist, and we don't
        # have a convenient way of making a directory in the output tree, so let's just use the
        # working directory as our output dir.
        # This also has the benefit of not needing to strip leading directories from the returned
        # snapshot.
        '-d', '.',
      ])
    else:
      javac_cmd.extend([
        '-d', ctx.classes_dir,
      ])

    javac_cmd.extend(self._javac_plugin_args(javac_plugin_map))

    javac_cmd.extend(args)

    if fatal_warnings:
      javac_cmd.extend(self.get_options().fatal_warnings_enabled_args)
    else:
      javac_cmd.extend(self.get_options().fatal_warnings_disabled_args)

    with argfile.safe_args(ctx.sources, self.get_options()) as batched_sources:
      javac_cmd.extend(batched_sources)

      if self.execution_strategy == self.HERMETIC:
        self._execute_hermetic_compile(javac_cmd, ctx)
      else:
        with self.context.new_workunit(name='javac',
                                       cmd=' '.join(javac_cmd),
                                       labels=[WorkUnitLabel.COMPILER]) as workunit:
          self.context.log.debug('Executing {}'.format(' '.join(javac_cmd)))
          p = subprocess.Popen(javac_cmd, stdout=workunit.output('stdout'), stderr=workunit.output('stderr'))
          return_code = p.wait()
          workunit.set_outcome(WorkUnit.FAILURE if return_code else WorkUnit.SUCCESS)
          if return_code:
            raise TaskError('javac exited with return code {rc}'.format(rc=return_code))
Exemplo n.º 24
0
    def console_output(self, targets):
        targets_map = {}
        resource_target_map = {}
        ivy_info = None
        if self.get_options().libraries:
            ivy_jar_products = self.context.products.get_data("ivy_jar_products") or {}
            # This product is a list for historical reasons (exclusives groups) but in practice should
            # have either 0 or 1 entries.
            ivy_info_list = ivy_jar_products.get("default")
            if ivy_info_list:
                assert len(ivy_info_list) == 1, (
                    "The values in ivy_jar_products should always be length 1,"
                    " since we no longer have exclusives groups."
                )
                ivy_info = ivy_info_list[0]

        ivy_jar_memo = {}
        python_interpreter_targets_mapping = defaultdict(list)

        def process_target(current_target):
            """
      :type current_target:pants.base.target.Target
      """

            def get_target_type(target):
                if target.is_test:
                    return Export.SourceRootTypes.TEST
                else:
                    if (
                        isinstance(target, Resources)
                        and target in resource_target_map
                        and resource_target_map[target].is_test
                    ):
                        return Export.SourceRootTypes.TEST_RESOURCE
                    elif isinstance(target, Resources):
                        return Export.SourceRootTypes.RESOURCE
                    else:
                        return Export.SourceRootTypes.SOURCE

            def get_transitive_jars(jar_lib):
                """
        :type jar_lib: pants.backend.jvm.targets.jar_library.JarLibrary
        :rtype: twitter.common.collections.orderedset.OrderedSet
        """
                if not ivy_info or not self.get_options().libraries:
                    return OrderedSet()
                transitive_jars = OrderedSet()
                for jar in jar_lib.jar_dependencies:
                    transitive_jars.update(ivy_info.get_jars_for_ivy_module(jar, memo=ivy_jar_memo))
                return transitive_jars

            info = {
                "targets": [],
                "libraries": [],
                "roots": [],
                "target_type": get_target_type(current_target),
                "is_code_gen": current_target.is_codegen,
                "pants_target_type": self._get_pants_target_alias(type(current_target)),
            }

            if not current_target.is_synthetic:
                info["globs"] = current_target.globs_relative_to_buildroot()
                if self.get_options().sources:
                    info["sources"] = list(current_target.sources_relative_to_buildroot())

            if isinstance(current_target, PythonRequirementLibrary):
                reqs = current_target.payload.get_field_value("requirements", set())
                """:type : set[pants.backend.python.python_requirement.PythonRequirement]"""
                info["requirements"] = [req.key for req in reqs]

            if isinstance(current_target, PythonTarget):
                interpreter_for_target = self.select_interpreter_for_targets([current_target])
                if interpreter_for_target is None:
                    raise TaskError("Unable to find suitable interpreter for {}".format(current_target.address))
                python_interpreter_targets_mapping[interpreter_for_target].append(current_target)
                info["python_interpreter"] = str(interpreter_for_target.identity)

            target_libraries = OrderedSet()
            if isinstance(current_target, JarLibrary):
                target_libraries = get_transitive_jars(current_target)
            for dep in current_target.dependencies:
                info["targets"].append(dep.address.spec)
                if isinstance(dep, JarLibrary):
                    for jar in dep.jar_dependencies:
                        target_libraries.add(IvyModuleRef(jar.org, jar.name, jar.rev))
                    # Add all the jars pulled in by this jar_library
                    target_libraries.update(get_transitive_jars(dep))
                if isinstance(dep, Resources):
                    resource_target_map[dep] = current_target

            if isinstance(current_target, ScalaLibrary):
                for dep in current_target.java_sources:
                    info["targets"].append(dep.address.spec)
                    process_target(dep)

            if isinstance(current_target, JvmTarget):
                info["excludes"] = [self._exclude_id(exclude) for exclude in current_target.excludes]
                info["platform"] = current_target.platform.name

            info["roots"] = map(
                lambda (source_root, package_prefix): {"source_root": source_root, "package_prefix": package_prefix},
                self._source_roots_for_target(current_target),
            )

            if self.get_options().libraries:
                info["libraries"] = [self._jar_id(lib) for lib in target_libraries]
            targets_map[current_target.address.spec] = info

        for target in targets:
            process_target(target)

        jvm_platforms_map = {
            "default_platform": JvmPlatform.global_instance().default_platform.name,
            "platforms": {
                str(platform_name): {
                    "target_level": str(platform.target_level),
                    "source_level": str(platform.source_level),
                    "args": platform.args,
                }
                for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items()
            },
        }

        graph_info = {
            "version": self.DEFAULT_EXPORT_VERSION,
            "targets": targets_map,
            "jvm_platforms": jvm_platforms_map,
        }
        jvm_distributions = DistributionLocator.global_instance().all_jdk_paths()
        if jvm_distributions:
            graph_info["jvm_distributions"] = jvm_distributions

        if self.get_options().libraries:
            graph_info["libraries"] = self._resolve_jars_info()

        if python_interpreter_targets_mapping:
            default_interpreter = self.interpreter_cache.select_interpreter(python_interpreter_targets_mapping.keys())[
                0
            ]

            interpreters_info = {}
            for interpreter, targets in python_interpreter_targets_mapping.iteritems():
                chroot = self.cached_chroot(interpreter=interpreter, pex_info=PexInfo.default(), targets=targets)
                interpreters_info[str(interpreter.identity)] = {"binary": interpreter.binary, "chroot": chroot.path()}

            graph_info["python_setup"] = {
                "default_interpreter": str(default_interpreter.identity),
                "interpreters": interpreters_info,
            }

        if self.format:
            return json.dumps(graph_info, indent=4, separators=(",", ": ")).splitlines()
        else:
            return [json.dumps(graph_info)]
Exemplo n.º 25
0
  def _run_tests(self, tests_to_targets):
    if self._coverage:
      extra_jvm_options = self._coverage.extra_jvm_options
      classpath_prepend = self._coverage.classpath_prepend
      classpath_append = self._coverage.classpath_append
    else:
      extra_jvm_options = []
      classpath_prepend = ()
      classpath_append = ()

    tests_by_properties = self._tests_by_properties(
      tests_to_targets,
      self._infer_workdir,
      lambda target: target.test_platform,
      lambda target: target.payload.extra_jvm_options,
      lambda target: target.payload.extra_env_vars,
      lambda target: target.concurrency,
      lambda target: target.threads
    )

    # the below will be None if not set, and we'll default back to runtime_classpath
    classpath_product = self.context.products.get_data('instrument_classpath')

    result = 0
    for properties, tests in tests_by_properties.items():
      (workdir, platform, target_jvm_options, target_env_vars, concurrency, threads) = properties
      for batch in self._partition(tests):
        # Batches of test classes will likely exist within the same targets: dedupe them.
        relevant_targets = set(map(tests_to_targets.get, batch))
        complete_classpath = OrderedSet()
        complete_classpath.update(classpath_prepend)
        complete_classpath.update(self.tool_classpath('junit'))
        complete_classpath.update(self.classpath(relevant_targets,
                                                 classpath_product=classpath_product))
        complete_classpath.update(classpath_append)
        distribution = JvmPlatform.preferred_jvm_distribution([platform], self._strict_jvm_version)

        # Override cmdline args with values from junit_test() target that specify concurrency:
        args = self._args + [u'-xmlreport']

        if concurrency is not None:
          args = remove_arg(args, '-default-parallel')
          if concurrency == junit_tests.CONCURRENCY_SERIAL:
            args = ensure_arg(args, '-default-concurrency', param='SERIAL')
          elif concurrency == junit_tests.CONCURRENCY_PARALLEL_CLASSES:
            args = ensure_arg(args, '-default-concurrency', param='PARALLEL_CLASSES')
          elif concurrency == junit_tests.CONCURRENCY_PARALLEL_METHODS:
            args = ensure_arg(args, '-default-concurrency', param='PARALLEL_METHODS')
          elif concurrency == junit_tests.CONCURRENCY_PARALLEL_CLASSES_AND_METHODS:
            args = ensure_arg(args, '-default-concurrency', param='PARALLEL_CLASSES_AND_METHODS')

        if threads is not None:
          args = remove_arg(args, '-parallel-threads', has_param=True)
          args += ['-parallel-threads', str(threads)]

        with argfile.safe_args(batch, self.get_options()) as batch_tests:
          self.context.log.debug('CWD = {}'.format(workdir))
          self.context.log.debug('platform = {}'.format(platform))
          with environment_as(**dict(target_env_vars)):
            result += abs(self._spawn_and_wait(
              executor=SubprocessExecutor(distribution),
              distribution=distribution,
              classpath=complete_classpath,
              main=JUnitRun._MAIN,
              jvm_options=self.jvm_options + extra_jvm_options + list(target_jvm_options),
              args=args + batch_tests,
              workunit_factory=self.context.new_workunit,
              workunit_name='run',
              workunit_labels=[WorkUnitLabel.TEST],
              cwd=workdir,
              synthetic_jar_dir=self.workdir,
              create_synthetic_jar=self.synthetic_classpath,
            ))

          if result != 0 and self._fail_fast:
            break

    if result != 0:
      failed_targets_and_tests = self._get_failed_targets(tests_to_targets)
      failed_targets = sorted(failed_targets_and_tests, key=lambda target: target.address.spec)
      error_message_lines = []
      if self._failure_summary:
        for target in failed_targets:
          error_message_lines.append('\n{0}{1}'.format(' '*4, target.address.spec))
          for test in sorted(failed_targets_and_tests[target]):
            error_message_lines.append('{0}{1}'.format(' '*8, test))
      error_message_lines.append(
        '\njava {main} ... exited non-zero ({code}); {failed} failed {targets}.'
          .format(main=JUnitRun._MAIN, code=result, failed=len(failed_targets),
                  targets=pluralize(len(failed_targets), 'target'))
      )
      raise TestFailedTaskError('\n'.join(error_message_lines), failed_targets=list(failed_targets))
Exemplo n.º 26
0
 def execute(self):
   if JvmPlatform.global_instance().get_options().compiler == 'javac':
     return super(JavacCompile, self).execute()
Exemplo n.º 27
0
    def test_strict_usage(self):
        init_subsystem(
            JvmPlatform,
            options={
                "jvm-platform": {
                    "platforms": {
                        "default-platform": {"target": "9"},
                        "8-platform": {"target": "8"},
                        "9-platform": {"target": "9"},
                        "strict-8-platform": {"target": "8", "strict": True},
                        "strict-9-platform": {"target": "9", "strict": True},
                    },
                    "default_platform": "default-platform",
                    "default_runtime_platform": None,
                }
            },
        )
        instance = JvmPlatform.global_instance()
        strict_8_platform = instance.get_platform_by_name("strict-8-platform")
        default_9_platform = instance.default_platform
        # TODO maybe this should use the runtime platform
        assert instance._preferred_jvm_distribution_args([]) == {
            "jdk": False,
        }
        assert JvmPlatform._preferred_jvm_distribution_args([default_9_platform]) == {
            "minimum_version": Revision.lenient("9.0.0"),
            "maximum_version": None,
            "jdk": False,
        }
        assert JvmPlatform._preferred_jvm_distribution_args([default_9_platform], strict=True) == {
            "minimum_version": Revision.lenient("9.0.0"),
            "maximum_version": Revision.lenient("9.0.9999"),
            "jdk": False,
        }
        assert instance._preferred_jvm_distribution_args([strict_8_platform]) == {
            "minimum_version": Revision.lenient("1.8.0"),
            "maximum_version": Revision.lenient("1.8.9999"),
            "jdk": False,
        }
        assert instance._preferred_jvm_distribution_args([strict_8_platform], strict=False) == {
            "minimum_version": Revision.lenient("1.8.0"),
            "maximum_version": None,
            "jdk": False,
        }

        with self.assertRaisesRegex(
            JvmPlatform.IncompatiblePlatforms,
            "lenient platform with higher minimum version, 9, than strict requirement of 1.8",
        ):
            # requested strict 8 & lenient 9.
            # fail because 9 is lower bound
            JvmPlatform._preferred_jvm_distribution_args(
                [
                    instance.get_platform_by_name("9-platform"),
                    instance.get_platform_by_name("strict-8-platform"),
                ]
            )
        with self.assertRaisesRegex(
            JvmPlatform.IncompatiblePlatforms,
            "Multiple strict platforms with differing target releases were found: 1.8, 9",
        ):
            # two different strict platforms can't work
            JvmPlatform._preferred_jvm_distribution_args(
                [
                    instance.get_platform_by_name("strict-9-platform"),
                    instance.get_platform_by_name("strict-8-platform"),
                ]
            )
        # two of the same strict platform thumbs up
        assert JvmPlatform._preferred_jvm_distribution_args(
            [
                instance.get_platform_by_name("strict-8-platform"),
                instance.get_platform_by_name("strict-8-platform"),
            ]
        ) == {
            "minimum_version": Revision.lenient("1.8.0"),
            "maximum_version": Revision.lenient("1.8.9999"),
            "jdk": False,
        }
        # strict highest, matching highest non-strict, other non-strict
        assert JvmPlatform._preferred_jvm_distribution_args(
            [
                instance.get_platform_by_name("strict-9-platform"),
                instance.get_platform_by_name("9-platform"),
                instance.get_platform_by_name("8-platform"),
            ]
        ) == {
            "minimum_version": Revision.lenient("9.0.0"),
            "maximum_version": Revision.lenient("9.0.9999"),
            "jdk": False,
        }
    def generate_project(self, project):
        def linked_folder_id(source_set):
            return source_set.source_base.replace(os.path.sep, '.')

        def base_path(source_set):
            return os.path.join(source_set.root_dir, source_set.source_base,
                                source_set.path)

        def create_source_base_template(source_set):
            source_base = base_path(source_set)
            return SourceBase(id=linked_folder_id(source_set),
                              path=source_base)

        source_sets = project.sources[:]
        if project.has_python:
            source_sets.extend(project.py_sources)

        source_bases = frozenset(map(create_source_base_template, source_sets))

        libs = []

        def add_jarlibs(classpath_entries):
            for classpath_entry in classpath_entries:
                libs.append((classpath_entry.jar, classpath_entry.source_jar))

        add_jarlibs(project.internal_jars)
        add_jarlibs(project.external_jars)
        scala_full_version = scala_platform.scala_build_info[
            self.context.options['scala-platform']['version']].full_version
        scala = {
            'language_level': scala_full_version,
            'compiler_classpath': project.scala_compiler_classpath
        }

        outdir = os.path.abspath(self.ensime_output_dir)
        if not os.path.exists(outdir):
            os.makedirs(outdir)

        java_platform = JvmPlatform.global_instance().default_platform
        jdk_home = JvmPlatform.preferred_jvm_distribution([java_platform],
                                                          strict=True).home

        configured_project = {
            'name':
            self.project_name,
            'java_home':
            jdk_home,
            'scala':
            scala,
            'source_bases':
            source_bases,
            'has_tests':
            project.has_tests,
            'internal_jars':
            [cp_entry.jar for cp_entry in project.internal_jars],
            'internal_source_jars': [
                cp_entry.source_jar for cp_entry in project.internal_jars
                if cp_entry.source_jar
            ],
            'external_jars':
            [cp_entry.jar for cp_entry in project.external_jars],
            'external_javadoc_jars': [
                cp_entry.javadoc_jar for cp_entry in project.external_jars
                if cp_entry.javadoc_jar
            ],
            'external_source_jars': [
                cp_entry.source_jar for cp_entry in project.external_jars
                if cp_entry.source_jar
            ],
            'libs':
            libs,
            'outdir':
            os.path.relpath(outdir, get_buildroot()),
            'root_dir':
            get_buildroot(),
            'cache_dir':
            os.path.join(self.cwd, '.ensime_cache')
        }

        with safe_open(self.project_filename, 'w') as output:
            output.write(
                self._renderer.render_name(_TEMPLATE,
                                           {'project': configured_project}))

        print('\nGenerated ensime project at {}{}'.format(
            self.gen_project_workdir, os.sep))
Exemplo n.º 29
0
  def generate_targets_map(self, targets, runtime_classpath):
    """Generates a dictionary containing all pertinent information about the target graph.

    The return dictionary is suitable for serialization by json.dumps.
    :param targets: The list of targets to generate the map for.
    :param classpath_products: Optional classpath_products. If not provided when the --libraries
      option is `True`, this task will perform its own jar resolution.
    """

    def _get_target_type(tgt):
      def is_test(t):
        return isinstance(t, JUnitTests)

      if is_test(tgt):
        return SourceRootTypes.TEST
      else:
        if (isinstance(tgt, Resources) and
          tgt in resource_target_map and
          is_test(resource_target_map[tgt])):
          return SourceRootTypes.TEST_RESOURCE
        elif isinstance(tgt, Resources):
          return SourceRootTypes.RESOURCE
        else:
          return SourceRootTypes.SOURCE

    targets_map = {}
    resource_target_map = {}
    target_roots_set = set(self.context.target_roots)

    def process_target(current_target):
      """
      :type current_target:pants.build_graph.target.Target
      """
      info = {
        # this means 'dependencies'
        'targets': [],
        'libraries': [],
        'roots': [],
        'id': current_target.id,
        'target_type': _get_target_type(current_target),
        'is_synthetic': current_target.is_synthetic,
        'pants_target_type': self._get_pants_target_alias(type(current_target)),
        'is_target_root': current_target in target_roots_set,
        'transitive': current_target.transitive,
        'scope': str(current_target.scope)
      }

      if not current_target.is_synthetic:
        info['globs'] = current_target.globs_relative_to_buildroot()

      def iter_transitive_jars(jar_lib):
        """
        :type jar_lib: :class:`pants.backend.jvm.targets.jar_library.JarLibrary`
        :rtype: :class:`collections.Iterator` of
                :class:`pants.java.jar.M2Coordinate`
        """
        if runtime_classpath:
          jar_products = runtime_classpath.get_artifact_classpath_entries_for_targets((jar_lib,))
          for _, jar_entry in jar_products:
            coordinate = jar_entry.coordinate
            # We drop classifier and type_ since those fields are represented in the global
            # libraries dict and here we just want the key into that dict (see `_jar_id`).
            yield M2Coordinate(org=coordinate.org, name=coordinate.name, rev=coordinate.rev)

      target_libraries = OrderedSet()
      if isinstance(current_target, JarLibrary):
        target_libraries = OrderedSet(iter_transitive_jars(current_target))
      for dep in current_target.dependencies:
        info['targets'].append(dep.address.spec)
        if isinstance(dep, JarLibrary):
          for jar in dep.jar_dependencies:
            target_libraries.add(M2Coordinate(jar.org, jar.name, jar.rev))
          # Add all the jars pulled in by this jar_library
          target_libraries.update(iter_transitive_jars(dep))
        if isinstance(dep, Resources):
          resource_target_map[dep] = current_target

      if isinstance(current_target, ScalaLibrary):
        for dep in current_target.java_sources:
          info['targets'].append(dep.address.spec)

      if isinstance(current_target, JvmTarget):
        info['excludes'] = [self._exclude_id(exclude) for exclude in current_target.excludes]
        info['platform'] = current_target.platform.name
        if hasattr(current_target, 'test_platform'):
          info['test_platform'] = current_target.test_platform.name

      if runtime_classpath:
        info['libraries'].extend(self._jar_id(lib) for lib in target_libraries)

      if current_target in target_roots_set:
        info['roots'] = [{
          'source_root': os.path.realpath(source_root_package_prefix[0]),
          'package_prefix': source_root_package_prefix[1]
        } for source_root_package_prefix in self._source_roots_for_target(current_target)]

      targets_map[current_target.address.spec] = info

    additional_java_targets = []
    for t in targets:
      if isinstance(t, ScalaLibrary):
        additional_java_targets.extend(t.java_sources)

    targets.extend(additional_java_targets)

    for target in targets:
      process_target(target)

    scala_platform = ScalaPlatform.global_instance()
    scala_platform_map = {
      'scala_version': scala_platform.version,
      'compiler_classpath': [
        cp_entry.path
        for cp_entry in scala_platform.compiler_classpath_entries(self.context.products)
      ],
    }

    jvm_platforms_map = {
      'default_platform': JvmPlatform.global_instance().default_platform.name,
      'platforms': {
        str(platform_name): {
          'target_level': str(platform.target_level),
          'source_level': str(platform.source_level),
          'args': platform.args,
        } for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items()},
    }

    graph_info = {
      'version': DEFAULT_EXPORT_VERSION,
      'targets': targets_map,
      'jvm_platforms': jvm_platforms_map,
      'scala_platform': scala_platform_map,
      # `jvm_distributions` are static distribution settings from config,
      # `preferred_jvm_distributions` are distributions that pants actually uses for the
      # given platform setting.
      'preferred_jvm_distributions': {}
    }

    for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items():
      preferred_distributions = {}
      for strict, strict_key in [(True, 'strict'), (False, 'non_strict')]:
        try:
          dist = JvmPlatform.preferred_jvm_distribution([platform], strict=strict)
          preferred_distributions[strict_key] = dist.home
        except DistributionLocator.Error:
          pass

      if preferred_distributions:
        graph_info['preferred_jvm_distributions'][platform_name] = preferred_distributions

    def zip_sources(target, location, suffix='.jar'):
      with temporary_file(root_dir=location, cleanup=False, suffix=suffix) as f:
        with zipfile.ZipFile(f, 'a') as zip_file:
          for src_from_source_root, src_from_build_root in zip(target.sources_relative_to_source_root(), target.sources_relative_to_buildroot()):
            zip_file.write(os.path.join(get_buildroot(), src_from_build_root), src_from_source_root)
      return f

    if runtime_classpath:
      graph_info['libraries'] = self._resolve_jars_info(targets, runtime_classpath)
      # Using resolved path in preparation for VCFS.
      resource_jar_root = os.path.realpath(self.versioned_workdir)
      for t in targets:
        target_type = _get_target_type(t)
        # If it is a target root or it is already a jar_library target, then no-op.
        if t in target_roots_set or targets_map[t.address.spec]['pants_target_type'] == 'jar_library':
          continue

        if target_type == SourceRootTypes.RESOURCE or target_type == SourceRootTypes.TEST_RESOURCE:
          # yic assumed that the cost to fingerprint the target may not be that lower than
          # just zipping up the resources anyway.
          jarred_resources = zip_sources(t, resource_jar_root)
          targets_map[t.address.spec]['pants_target_type'] = 'jar_library'
          targets_map[t.address.spec]['libraries'] = [t.id]
          graph_info['libraries'][t.id]['default'] = jarred_resources.name
          continue

        targets_map[t.address.spec]['pants_target_type'] = 'jar_library'
        targets_map[t.address.spec]['libraries'] = [t.id]
        jar_products = runtime_classpath.get_for_target(t)
        for conf, jar_entry in jar_products:
          # TODO(yic): check --compile-rsc-use-classpath-jars is enabled.
          # If not, zip up the classes/ dir here.
          if 'z.jar' in jar_entry:
            graph_info['libraries'][t.id][conf] = jar_entry
        if self.get_options().sources:
          # NB: We create the jar in the same place as we create the resources
          # (as opposed to where we store the z.jar), because the path to the z.jar depends
          # on tasks outside of this one.
          # In addition to that, we may not want to depend on z.jar existing to export source jars.
          jarred_sources = zip_sources(t, resource_jar_root, suffix='-sources.jar')
          graph_info['libraries'][t.id]['sources'] = jarred_sources.name


    return graph_info
Exemplo n.º 30
0
    def console_output(self, targets):
        targets_map = {}
        resource_target_map = {}
        ivy_info = None
        if self.get_options().libraries:
            ivy_jar_products = self.context.products.get_data(
                'ivy_jar_products') or {}
            # This product is a list for historical reasons (exclusives groups) but in practice should
            # have either 0 or 1 entries.
            ivy_info_list = ivy_jar_products.get('default')
            if ivy_info_list:
                assert len(ivy_info_list) == 1, (
                    'The values in ivy_jar_products should always be length 1,'
                    ' since we no longer have exclusives groups.')
                ivy_info = ivy_info_list[0]

        ivy_jar_memo = {}
        python_interpreter_targets_mapping = defaultdict(list)

        def process_target(current_target):
            """
      :type current_target:pants.base.target.Target
      """
            def get_target_type(target):
                if target.is_test:
                    return Export.SourceRootTypes.TEST
                else:
                    if (isinstance(target, Resources)
                            and target in resource_target_map
                            and resource_target_map[target].is_test):
                        return Export.SourceRootTypes.TEST_RESOURCE
                    elif isinstance(target, Resources):
                        return Export.SourceRootTypes.RESOURCE
                    else:
                        return Export.SourceRootTypes.SOURCE

            def get_transitive_jars(jar_lib):
                """
        :type jar_lib: pants.backend.jvm.targets.jar_library.JarLibrary
        :rtype: twitter.common.collections.orderedset.OrderedSet
        """
                if not ivy_info or not self.get_options().libraries:
                    return OrderedSet()
                transitive_jars = OrderedSet()
                for jar in jar_lib.jar_dependencies:
                    transitive_jars.update(
                        ivy_info.get_jars_for_ivy_module(jar,
                                                         memo=ivy_jar_memo))
                return transitive_jars

            info = {
                'targets': [],
                'libraries': [],
                'roots': [],
                'target_type':
                get_target_type(current_target),
                'is_code_gen':
                current_target.is_codegen,
                'pants_target_type':
                self._get_pants_target_alias(type(current_target))
            }

            if not current_target.is_synthetic:
                info['globs'] = current_target.globs_relative_to_buildroot()
                if self.get_options().sources:
                    info['sources'] = list(
                        current_target.sources_relative_to_buildroot())

            if isinstance(current_target, PythonRequirementLibrary):
                reqs = current_target.payload.get_field_value(
                    'requirements', set())
                """:type : set[pants.backend.python.python_requirement.PythonRequirement]"""
                info['requirements'] = [req.key for req in reqs]

            if isinstance(current_target, PythonTarget):
                interpreter_for_target = self.select_interpreter_for_targets(
                    [current_target])
                if interpreter_for_target is None:
                    raise TaskError(
                        'Unable to find suitable interpreter for {}'.format(
                            current_target.address))
                python_interpreter_targets_mapping[
                    interpreter_for_target].append(current_target)
                info['python_interpreter'] = str(
                    interpreter_for_target.identity)

            target_libraries = OrderedSet()
            if isinstance(current_target, JarLibrary):
                target_libraries = get_transitive_jars(current_target)
            for dep in current_target.dependencies:
                info['targets'].append(dep.address.spec)
                if isinstance(dep, JarLibrary):
                    for jar in dep.jar_dependencies:
                        target_libraries.add(
                            IvyModuleRef(jar.org, jar.name, jar.rev))
                    # Add all the jars pulled in by this jar_library
                    target_libraries.update(get_transitive_jars(dep))
                if isinstance(dep, Resources):
                    resource_target_map[dep] = current_target

            if isinstance(current_target, ScalaLibrary):
                for dep in current_target.java_sources:
                    info['targets'].append(dep.address.spec)
                    process_target(dep)

            if isinstance(current_target, JvmTarget):
                info['excludes'] = [
                    self._exclude_id(exclude)
                    for exclude in current_target.excludes
                ]
                info['platform'] = current_target.platform.name

            info['roots'] = map(
                lambda (source_root, package_prefix): {
                    'source_root': source_root,
                    'package_prefix': package_prefix
                }, self._source_roots_for_target(current_target))

            if self.get_options().libraries:
                info['libraries'] = [
                    self._jar_id(lib) for lib in target_libraries
                ]
            targets_map[current_target.address.spec] = info

        for target in targets:
            process_target(target)

        jvm_platforms_map = {
            'default_platform':
            JvmPlatform.global_instance().default_platform.name,
            'platforms': {
                str(platform_name): {
                    'target_level': str(platform.target_level),
                    'source_level': str(platform.source_level),
                    'args': platform.args,
                }
                for platform_name, platform in
                JvmPlatform.global_instance().platforms_by_name.items()
            }
        }

        graph_info = {
            'version': self.DEFAULT_EXPORT_VERSION,
            'targets': targets_map,
            'jvm_platforms': jvm_platforms_map,
        }
        jvm_distributions = DistributionLocator.global_instance(
        ).all_jdk_paths()
        if jvm_distributions:
            graph_info['jvm_distributions'] = jvm_distributions

        if self.get_options().libraries:
            graph_info['libraries'] = self._resolve_jars_info()

        if python_interpreter_targets_mapping:
            default_interpreter = self.interpreter_cache.select_interpreter(
                python_interpreter_targets_mapping.keys())[0]

            interpreters_info = {}
            for interpreter, targets in python_interpreter_targets_mapping.iteritems(
            ):
                chroot = self.cached_chroot(interpreter=interpreter,
                                            pex_info=PexInfo.default(),
                                            targets=targets)
                interpreters_info[str(interpreter.identity)] = {
                    'binary': interpreter.binary,
                    'chroot': chroot.path()
                }

            graph_info['python_setup'] = {
                'default_interpreter': str(default_interpreter.identity),
                'interpreters': interpreters_info
            }

        if self.format:
            return json.dumps(graph_info, indent=4,
                              separators=(',', ': ')).splitlines()
        else:
            return [json.dumps(graph_info)]
Exemplo n.º 31
0
 def get_preferred_distribution(platform, strict):
     try:
         return JvmPlatform.preferred_jvm_distribution([platform],
                                                       strict=strict)
     except DistributionLocator.Error:
         return None
Exemplo n.º 32
0
  def _run_tests(self, test_registry, output_dir, coverage=None):

    tests_by_properties = test_registry.index(
      lambda tgt: tgt.cwd if tgt.cwd is not None else self._working_dir,
      lambda tgt: tgt.platform)

    # the below will be None if not set, and we'll default back to runtime_classpath
    classpath_product = self.context.products.get_data('instrument_classpath')

    result = 0
    base_args = self.get_options().args
    for properties, tests in tests_by_properties.items():
      (cwd, platform) = properties
      for batch in self._partition(tests, test_registry):
        # Batches of test classes will likely exist within the same targets: dedupe them.
        relevant_targets = {test_registry.get_owning_target(t) for t in batch}
        if len(relevant_targets) > 1:
          raise "oops, should have only had one target"
        complete_classpath = OrderedSet()
        # TODO: Include specs2 on the classpath, in case the target doesn't
        complete_classpath.update(self.classpath(relevant_targets,
                                                 classpath_product=classpath_product))
        distribution = JvmPlatform.preferred_jvm_distribution([platform], self._strict_jvm_version)

        target_dir = list(relevant_targets)[0].address.spec_path
        args = base_args[:]
        opts = self.get_options()
        if opts.example:
          args.extend(["ex", opts.example])
        if opts.sequential:
          args.extend(['sequential', 'true'])
        if opts.show_times:
          args.extend(['showtimes', 'true'])
        file_pattern = opts.file_pattern or Specs2Run.TEST_CLASS_PATTERN
        if '(' not in file_pattern or ')' not in file_pattern:
          raise TaskError("Test regex must have a group.")
        args.extend([
          "junitxml",
          "console",
          "filesrunner.basepath", target_dir,
          "filesrunner.pattern", file_pattern,
          "junit.outdir", self.junit_xml_dir
        ])
        self.context.log.debug('CWD = {}'.format(cwd))
        self.context.log.debug('platform = {}'.format(platform))
        self.context.log.debug('targets = {}'.format(relevant_targets))
        self.context.log.debug('args = {}'.format(" ".join(args)))
        result += abs(self._spawn_and_wait(
          executor=SubprocessExecutor(distribution),
          distribution=distribution,
          classpath=complete_classpath,
          main=Specs2Run.SPECS2_MAIN,
          jvm_options=self.jvm_options,
          args=args,
          workunit_factory=self.context.new_workunit,
          workunit_name='run',
          workunit_labels=[WorkUnitLabel.TEST],
          cwd=cwd,
          synthetic_jar_dir=output_dir,
          create_synthetic_jar=self.synthetic_classpath,
        ))

        if result != 0 and self._fail_fast:
          break

    if result != 0:
      failed_targets_and_tests = self._get_failed_targets(test_registry, output_dir)
      failed_targets = sorted(failed_targets_and_tests, key=lambda target: target.address.spec)
      error_message_lines = []
      if self._failure_summary:
        for target in failed_targets:
          error_message_lines.append('\n{0}{1}'.format(' '*4, target.address.spec))
          for test in sorted(failed_targets_and_tests[target]):
            error_message_lines.append('{0}{1}'.format(' '*8, test))
      error_message_lines.append(
        '\njava {main} ... exited non-zero ({code}); {failed} failed {targets}.'
          .format(main=Specs2Run.SPECS2_MAIN, code=result, failed=len(failed_targets),
                  targets=pluralize(len(failed_targets), 'target'))
      )
      raise TestFailedTaskError('\n'.join(error_message_lines), failed_targets=list(failed_targets))
Exemplo n.º 33
0
    def execute(self):
        requested_compiler = JvmPlatform.global_instance().get_options(
        ).compiler
        if requested_compiler != self.compiler_name:
            return
        deprecated_conditional(
            lambda: requested_compiler == self.Compiler.ZINC,
            removal_version='1.20.0.dev0',
            entity_description='Requested a deprecated compiler: [{}].'.format(
                requested_compiler),
            hint_message='Compiler will be defaulted to [{}].'.format(
                self.compiler_name))

        if requested_compiler == self.Compiler.ZINC and self.compiler_name == self.Compiler.RSC:
            # Issue a deprecation warning (above) and rewrite zinc to rsc, as zinc is being deprecated.
            JvmPlatform.global_instance().get_options().compiler = RankedValue(
                0, self.compiler_name)
        elif requested_compiler != self.compiler_name:
            # If the requested compiler is not the one supported by this task, log and abort
            self.context.log.debug(
                'Requested an unsupported compiler [{}], aborting'.format(
                    requested_compiler))
            return

        # In case we have no relevant targets and return early, create the requested product maps.
        self.create_empty_extra_products()

        relevant_targets = list(self.context.targets(predicate=self.select))

        if not relevant_targets:
            return

        # Clone the compile_classpath to the runtime_classpath.
        classpath_product = self.create_runtime_classpath()

        fingerprint_strategy = DependencyContext.global_instance(
        ).create_fingerprint_strategy(classpath_product)
        # Note, JVM targets are validated (`vts.update()`) as they succeed.  As a result,
        # we begin writing artifacts out to the cache immediately instead of waiting for
        # all targets to finish.
        with self.invalidated(relevant_targets,
                              invalidate_dependents=True,
                              fingerprint_strategy=fingerprint_strategy,
                              topological_order=True) as invalidation_check:

            compile_contexts = {
                vt.target: self.create_compile_context(vt.target,
                                                       vt.results_dir)
                for vt in invalidation_check.all_vts
            }

            self.do_compile(
                invalidation_check,
                compile_contexts,
                classpath_product,
            )

            if not self.get_options().use_classpath_jars:
                # Once compilation has completed, replace the classpath entry for each target with
                # its jar'd representation.
                for ccs in compile_contexts.values():
                    cc = self.select_runtime_context(ccs)
                    for conf in self._confs:
                        classpath_product.remove_for_target(
                            cc.target, [(conf, cc.classes_dir)])
                        classpath_product.add_for_target(
                            cc.target, [(conf, cc.jar_file)])
Exemplo n.º 34
0
  def generate_targets_map(self, targets, classpath_products=None):
    """Generates a dictionary containing all pertinent information about the target graph.

    The return dictionary is suitable for serialization by json.dumps.
    :param targets: The list of targets to generate the map for.
    :param classpath_products: Optional classpath_products. If not provided when the --libraries
      option is `True`, this task will perform its own jar resolution.
    """
    targets_map = {}
    resource_target_map = {}
    python_interpreter_targets_mapping = defaultdict(list)

    if self.get_options().libraries:
      # NB(gmalmquist): This supports mocking the classpath_products in tests.
      if classpath_products is None:
        classpath_products = self.resolve_jars(targets)
    else:
      classpath_products = None

    def process_target(current_target):
      """
      :type current_target:pants.build_graph.target.Target
      """
      def get_target_type(target):
        if target.is_test:
          return ExportTask.SourceRootTypes.TEST
        else:
          if (isinstance(target, Resources) and
              target in resource_target_map and
              resource_target_map[target].is_test):
            return ExportTask.SourceRootTypes.TEST_RESOURCE
          elif isinstance(target, Resources):
            return ExportTask.SourceRootTypes.RESOURCE
          else:
            return ExportTask.SourceRootTypes.SOURCE

      info = {
        'targets': [],
        'libraries': [],
        'roots': [],
        'id': current_target.id,
        'target_type': get_target_type(current_target),
        # NB: is_code_gen should be removed when export format advances to 1.1.0 or higher
        'is_code_gen': current_target.is_codegen,
        'is_synthetic': current_target.is_synthetic,
        'pants_target_type': self._get_pants_target_alias(type(current_target))
      }

      if not current_target.is_synthetic:
        info['globs'] = current_target.globs_relative_to_buildroot()
        if self.get_options().sources:
          info['sources'] = list(current_target.sources_relative_to_buildroot())

      if isinstance(current_target, PythonRequirementLibrary):
        reqs = current_target.payload.get_field_value('requirements', set())
        """:type : set[pants.backend.python.python_requirement.PythonRequirement]"""
        info['requirements'] = [req.key for req in reqs]

      if isinstance(current_target, PythonTarget):
        interpreter_for_target = self.select_interpreter_for_targets([current_target])
        if interpreter_for_target is None:
          raise TaskError('Unable to find suitable interpreter for {}'
                          .format(current_target.address))
        python_interpreter_targets_mapping[interpreter_for_target].append(current_target)
        info['python_interpreter'] = str(interpreter_for_target.identity)

      def iter_transitive_jars(jar_lib):
        """
        :type jar_lib: :class:`pants.backend.jvm.targets.jar_library.JarLibrary`
        :rtype: :class:`collections.Iterator` of
                :class:`pants.backend.jvm.jar_dependency_utils.M2Coordinate`
        """
        if classpath_products:
          jar_products = classpath_products.get_artifact_classpath_entries_for_targets((jar_lib,))
          for _, jar_entry in jar_products:
            coordinate = jar_entry.coordinate
            # We drop classifier and type_ since those fields are represented in the global
            # libraries dict and here we just want the key into that dict (see `_jar_id`).
            yield M2Coordinate(org=coordinate.org, name=coordinate.name, rev=coordinate.rev)

      target_libraries = OrderedSet()
      if isinstance(current_target, JarLibrary):
        target_libraries = OrderedSet(iter_transitive_jars(current_target))
      for dep in current_target.dependencies:
        info['targets'].append(dep.address.spec)
        if isinstance(dep, JarLibrary):
          for jar in dep.jar_dependencies:
            target_libraries.add(M2Coordinate(jar.org, jar.name, jar.rev))
          # Add all the jars pulled in by this jar_library
          target_libraries.update(iter_transitive_jars(dep))
        if isinstance(dep, Resources):
          resource_target_map[dep] = current_target

      if isinstance(current_target, ScalaLibrary):
        for dep in current_target.java_sources:
          info['targets'].append(dep.address.spec)
          process_target(dep)

      if isinstance(current_target, JvmTarget):
        info['excludes'] = [self._exclude_id(exclude) for exclude in current_target.excludes]
        info['platform'] = current_target.platform.name

      info['roots'] = map(lambda (source_root, package_prefix): {
        'source_root': source_root,
        'package_prefix': package_prefix
      }, self._source_roots_for_target(current_target))

      if classpath_products:
        info['libraries'] = [self._jar_id(lib) for lib in target_libraries]
      targets_map[current_target.address.spec] = info

    for target in targets:
      process_target(target)

    jvm_platforms_map = {
      'default_platform' : JvmPlatform.global_instance().default_platform.name,
      'platforms': {
        str(platform_name): {
          'target_level' : str(platform.target_level),
          'source_level' : str(platform.source_level),
          'args' : platform.args,
        } for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items() }
    }

    graph_info = {
      'version': self.DEFAULT_EXPORT_VERSION,
      'targets': targets_map,
      'jvm_platforms': jvm_platforms_map,
    }
    jvm_distributions = DistributionLocator.global_instance().all_jdk_paths()
    if jvm_distributions:
      graph_info['jvm_distributions'] = jvm_distributions

    if classpath_products:
      graph_info['libraries'] = self._resolve_jars_info(targets, classpath_products)

    if python_interpreter_targets_mapping:
      interpreters = self.interpreter_cache.select_interpreter(
        python_interpreter_targets_mapping.keys())
      default_interpreter = interpreters[0]

      interpreters_info = {}
      for interpreter, targets in six.iteritems(python_interpreter_targets_mapping):
        chroot = self.cached_chroot(
          interpreter=interpreter,
          pex_info=PexInfo.default(),
          targets=targets
        )
        interpreters_info[str(interpreter.identity)] = {
          'binary': interpreter.binary,
          'chroot': chroot.path()
        }

      graph_info['python_setup'] = {
        'default_interpreter': str(default_interpreter.identity),
        'interpreters': interpreters_info
      }

    return graph_info
Exemplo n.º 35
0
 def get_preferred_distribution(platform, strict):
   try:
     return JvmPlatform.preferred_jvm_distribution([platform], strict=strict)
   except DistributionLocator.Error:
     return None
Exemplo n.º 36
0
  def generate_targets_map(self, targets, classpath_products=None):
    """Generates a dictionary containing all pertinent information about the target graph.

    The return dictionary is suitable for serialization by json.dumps.
    :param targets: The list of targets to generate the map for.
    :param classpath_products: Optional classpath_products. If not provided when the --libraries
      option is `True`, this task will perform its own jar resolution.
    """
    targets_map = {}
    resource_target_map = {}
    python_interpreter_targets_mapping = defaultdict(list)

    if self.get_options().libraries:
      # NB(gmalmquist): This supports mocking the classpath_products in tests.
      if classpath_products is None:
        classpath_products = self.resolve_jars(targets)
    else:
      classpath_products = None

    def process_target(current_target):
      """
      :type current_target:pants.build_graph.target.Target
      """
      def get_target_type(target):
        if target.is_test:
          return ExportTask.SourceRootTypes.TEST
        else:
          if (isinstance(target, Resources) and
              target in resource_target_map and
              resource_target_map[target].is_test):
            return ExportTask.SourceRootTypes.TEST_RESOURCE
          elif isinstance(target, Resources):
            return ExportTask.SourceRootTypes.RESOURCE
          else:
            return ExportTask.SourceRootTypes.SOURCE

      info = {
        'targets': [],
        'libraries': [],
        'roots': [],
        'target_type': get_target_type(current_target),
        'is_code_gen': current_target.is_codegen,
        'pants_target_type': self._get_pants_target_alias(type(current_target))
      }

      if not current_target.is_synthetic:
        info['globs'] = current_target.globs_relative_to_buildroot()
        if self.get_options().sources:
          info['sources'] = list(current_target.sources_relative_to_buildroot())

      if isinstance(current_target, PythonRequirementLibrary):
        reqs = current_target.payload.get_field_value('requirements', set())
        """:type : set[pants.backend.python.python_requirement.PythonRequirement]"""
        info['requirements'] = [req.key for req in reqs]

      if isinstance(current_target, PythonTarget):
        interpreter_for_target = self.select_interpreter_for_targets([current_target])
        if interpreter_for_target is None:
          raise TaskError('Unable to find suitable interpreter for {}'
                          .format(current_target.address))
        python_interpreter_targets_mapping[interpreter_for_target].append(current_target)
        info['python_interpreter'] = str(interpreter_for_target.identity)

      def iter_transitive_jars(jar_lib):
        """
        :type jar_lib: :class:`pants.backend.jvm.targets.jar_library.JarLibrary`
        :rtype: :class:`collections.Iterator` of
                :class:`pants.backend.jvm.jar_dependency_utils.M2Coordinate`
        """
        if classpath_products:
          jar_products = classpath_products.get_artifact_classpath_entries_for_targets((jar_lib,))
          for _, jar_entry in jar_products:
            coordinate = jar_entry.coordinate
            # We drop classifier and type_ since those fields are represented in the global
            # libraries dict and here we just want the key into that dict (see `_jar_id`).
            yield M2Coordinate(org=coordinate.org, name=coordinate.name, rev=coordinate.rev)

      target_libraries = OrderedSet()
      if isinstance(current_target, JarLibrary):
        target_libraries = OrderedSet(iter_transitive_jars(current_target))
      for dep in current_target.dependencies:
        info['targets'].append(dep.address.spec)
        if isinstance(dep, JarLibrary):
          for jar in dep.jar_dependencies:
            target_libraries.add(M2Coordinate(jar.org, jar.name, jar.rev))
          # Add all the jars pulled in by this jar_library
          target_libraries.update(iter_transitive_jars(dep))
        if isinstance(dep, Resources):
          resource_target_map[dep] = current_target

      if isinstance(current_target, ScalaLibrary):
        for dep in current_target.java_sources:
          info['targets'].append(dep.address.spec)
          process_target(dep)

      if isinstance(current_target, JvmTarget):
        info['excludes'] = [self._exclude_id(exclude) for exclude in current_target.excludes]
        info['platform'] = current_target.platform.name

      info['roots'] = map(lambda (source_root, package_prefix): {
        'source_root': source_root,
        'package_prefix': package_prefix
      }, self._source_roots_for_target(current_target))

      if classpath_products:
        info['libraries'] = [self._jar_id(lib) for lib in target_libraries]
      targets_map[current_target.address.spec] = info

    for target in targets:
      process_target(target)

    jvm_platforms_map = {
      'default_platform' : JvmPlatform.global_instance().default_platform.name,
      'platforms': {
        str(platform_name): {
          'target_level' : str(platform.target_level),
          'source_level' : str(platform.source_level),
          'args' : platform.args,
        } for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items() }
    }

    graph_info = {
      'version': self.DEFAULT_EXPORT_VERSION,
      'targets': targets_map,
      'jvm_platforms': jvm_platforms_map,
    }
    jvm_distributions = DistributionLocator.global_instance().all_jdk_paths()
    if jvm_distributions:
      graph_info['jvm_distributions'] = jvm_distributions

    if classpath_products:
      graph_info['libraries'] = self._resolve_jars_info(targets, classpath_products)

    if python_interpreter_targets_mapping:
      interpreters = self.interpreter_cache.select_interpreter(
        python_interpreter_targets_mapping.keys())
      default_interpreter = interpreters[0]

      interpreters_info = {}
      for interpreter, targets in six.iteritems(python_interpreter_targets_mapping):
        chroot = self.cached_chroot(
          interpreter=interpreter,
          pex_info=PexInfo.default(),
          targets=targets
        )
        interpreters_info[str(interpreter.identity)] = {
          'binary': interpreter.binary,
          'chroot': chroot.path()
        }

      graph_info['python_setup'] = {
        'default_interpreter': str(default_interpreter.identity),
        'interpreters': interpreters_info
      }

    return graph_info
Exemplo n.º 37
0
    def generate_project(self, project):
        def linked_folder_id(source_set):
            return source_set.source_base.replace(os.path.sep, '.')

        def base_path(source_set):
            return os.path.join(source_set.root_dir, source_set.source_base,
                                source_set.path)

        def create_source_base_template(source_set):
            source_base = base_path(source_set)
            return SourceBase(id=linked_folder_id(source_set),
                              path=source_base)

        source_sets = project.sources[:]
        if project.has_python:
            source_sets.extend(project.py_sources)

        source_bases = frozenset(map(create_source_base_template, source_sets))

        libs = []

        def add_jarlibs(classpath_entries):
            for classpath_entry in classpath_entries:
                libs.append((classpath_entry.jar, classpath_entry.source_jar))

        add_jarlibs(project.internal_jars)
        add_jarlibs(project.external_jars)
        scala_full_version = scala_platform.scala_build_info[
            self.context.options['scala-platform']['version']].full_version
        scala = TemplateData(
            language_level=scala_full_version,
            compiler_classpath=project.scala_compiler_classpath)

        outdir = os.path.abspath(self.ensime_output_dir)
        if not os.path.exists(outdir):
            os.makedirs(outdir)

        java_platform = JvmPlatform.global_instance().default_platform
        jdk_home = JvmPlatform.preferred_jvm_distribution([java_platform],
                                                          strict=True).home

        configured_project = TemplateData(
            name=self.project_name,
            java_home=jdk_home,
            scala=scala,
            source_bases=source_bases,
            has_tests=project.has_tests,
            internal_jars=[cp_entry.jar for cp_entry in project.internal_jars],
            internal_source_jars=[
                cp_entry.source_jar for cp_entry in project.internal_jars
                if cp_entry.source_jar
            ],
            external_jars=[cp_entry.jar for cp_entry in project.external_jars],
            external_javadoc_jars=[
                cp_entry.javadoc_jar for cp_entry in project.external_jars
                if cp_entry.javadoc_jar
            ],
            external_source_jars=[
                cp_entry.source_jar for cp_entry in project.external_jars
                if cp_entry.source_jar
            ],
            libs=libs,
            outdir=os.path.relpath(outdir, get_buildroot()),
            root_dir=get_buildroot(),
            cache_dir=os.path.join(self.cwd, '.ensime_cache'))

        def apply_template(output_path, template_relpath, **template_data):
            with safe_open(output_path, 'w') as output:
                Generator(pkgutil.get_data(__name__, template_relpath),
                          **template_data).write(output)

        apply_template(self.project_filename,
                       self.project_template,
                       project=configured_project)
        print('\nGenerated ensime project at {}{}'.format(
            self.gen_project_workdir, os.sep))
Exemplo n.º 38
0
  def compile(self, ctx, args, dependency_classpath, upstream_analysis,
              settings, compiler_option_sets, zinc_file_manager,
              javac_plugin_map, scalac_plugin_map):
    classpath = (ctx.classes_dir.path,) + tuple(ce.path for ce in dependency_classpath)

    if self.get_options().capture_classpath:
      self._record_compile_classpath(classpath, ctx.target, ctx.classes_dir.path)

    try:
      distribution = JvmPlatform.preferred_jvm_distribution([settings], strict=True)
    except DistributionLocator.Error:
      distribution = JvmPlatform.preferred_jvm_distribution([settings], strict=False)

    javac_args = []

    if settings.args:
      settings_args = settings.args
      if any('$JAVA_HOME' in a for a in settings.args):
        logger.debug('Substituting "$JAVA_HOME" with "{}" in jvm-platform args.'
                     .format(distribution.home))
        settings_args = (a.replace('$JAVA_HOME', distribution.home) for a in settings.args)
      javac_args.extend(settings_args)

      javac_args.extend([
        # TODO: support -release
        '-source', str(settings.source_level),
        '-target', str(settings.target_level),
      ])

    if self.execution_strategy == self.HERMETIC:
      javac_args.extend([
        # We need to strip the source root from our output files. Outputting to a directory, and
        # capturing that directory, does the job.
        # Unfortunately, javac errors if the directory you pass to -d doesn't exist, and we don't
        # have a convenient way of making a directory in the output tree, so let's just use the
        # working directory as our output dir.
        # This also has the benefit of not needing to strip leading directories from the returned
        # snapshot.
        '-d', '.',
      ])
    else:
      javac_args.extend([
        '-d', ctx.classes_dir.path,
      ])

    javac_args.extend(self._javac_plugin_args(javac_plugin_map))

    javac_args.extend(args)

    compiler_option_sets_args = self.get_merged_args_for_compiler_option_sets(compiler_option_sets)
    javac_args.extend(compiler_option_sets_args)

    javac_args.extend([
      '-classpath', ':'.join(classpath),
    ])
    javac_args.extend(ctx.sources)

    # From https://docs.oracle.com/javase/8/docs/technotes/tools/windows/javac.html#BHCJEIBB
    # Wildcards (*) aren’t allowed in these lists (such as for specifying *.java).
    # Use of the at sign (@) to recursively interpret files isn’t supported.
    # The -J options aren’t supported because they’re passed to the launcher,
    # which doesn’t support argument files.
    j_args = [j_arg for j_arg in javac_args if j_arg.startswith('-J')]
    safe_javac_args = list(filter(lambda x: x not in j_args, javac_args))

    with argfile.safe_args(safe_javac_args, self.get_options()) as batched_args:
      javac_cmd = ['{}/bin/javac'.format(distribution.real_home)]
      javac_cmd.extend(j_args)
      javac_cmd.extend(batched_args)

      if self.execution_strategy == self.HERMETIC:
        self._execute_hermetic_compile(javac_cmd, ctx)
      else:
        with self.context.new_workunit(name='javac',
                                       cmd=' '.join(javac_cmd),
                                       labels=[WorkUnitLabel.COMPILER]) as workunit:
          self.context.log.debug('Executing {}'.format(' '.join(javac_cmd)))
          p = subprocess.Popen(javac_cmd, stdout=workunit.output('stdout'), stderr=workunit.output('stderr'))
          return_code = p.wait()
          workunit.set_outcome(WorkUnit.FAILURE if return_code else WorkUnit.SUCCESS)
          if return_code:
            raise TaskError('javac exited with return code {rc}'.format(rc=return_code))
Exemplo n.º 39
0
 def preferred_jvm_distribution_for_targets(self, targets):
     return JvmPlatform.preferred_jvm_distribution([
         target.platform
         for target in targets if isinstance(target, JvmTarget)
     ], self._strict_jvm_version)
Exemplo n.º 40
0
 def preferred_jvm_distribution(self, platforms, jdk=False, strict=None):
     return JvmPlatform.preferred_jvm_distribution(platforms,
                                                   jdk=jdk,
                                                   strict=strict)
Exemplo n.º 41
0
 def preferred_jvm_distribution_for_targets(self, targets):
   return JvmPlatform.preferred_jvm_distribution([target.platform for target in targets
                                                 if isinstance(target, JvmTarget)],
                                                 self._strict_jvm_version)
Exemplo n.º 42
0
  def _run_tests(self, tests_to_targets):
    if self._coverage:
      extra_jvm_options = self._coverage.extra_jvm_options
      classpath_prepend = self._coverage.classpath_prepend
      classpath_append = self._coverage.classpath_append
    else:
      extra_jvm_options = []
      classpath_prepend = ()
      classpath_append = ()

    tests_by_properties = self._tests_by_properties(
      tests_to_targets,
      self._infer_workdir,
      lambda target: target.test_platform,
      lambda target: target.payload.extra_jvm_options,
      lambda target: target.payload.extra_env_vars,
      lambda target: target.concurrency,
      lambda target: target.threads
    )

    # the below will be None if not set, and we'll default back to runtime_classpath
    classpath_product = self.context.products.get_data('instrument_classpath')

    result = 0
    for properties, tests in tests_by_properties.items():
      (workdir, platform, target_jvm_options, target_env_vars, concurrency, threads) = properties
      for batch in self._partition(tests):
        # Batches of test classes will likely exist within the same targets: dedupe them.
        relevant_targets = set(map(tests_to_targets.get, batch))
        complete_classpath = OrderedSet()
        complete_classpath.update(classpath_prepend)
        complete_classpath.update(self.tool_classpath('junit'))
        complete_classpath.update(self.classpath(relevant_targets,
                                                 classpath_product=classpath_product))
        complete_classpath.update(classpath_append)
        distribution = JvmPlatform.preferred_jvm_distribution([platform], self._strict_jvm_version)

        # Override cmdline args with values from junit_test() target that specify concurrency:
        args = self._args + [u'-xmlreport']

        # TODO(zundel): Combine these together into a single -concurrency choices style argument
        if concurrency == junit_tests.CONCURRENCY_SERIAL:
          args = remove_arg(args, '-default-parallel')
        elif concurrency == junit_tests.CONCURRENCY_PARALLEL_CLASSES:
          args = ensure_arg(args, '-default-parallel')
        elif concurrency == junit_tests.CONCURRENCY_PARALLEL_METHODS:
          self.context.log.warn('Not implemented: parallel_methods')
        elif concurrency == junit_tests.CONCURRENCY_PARALLEL_BOTH:
          self.context.log.warn('specifying {} is experimental.'.format(concurrency))
          args = ensure_arg(args, '-default-parallel')
          args = ensure_arg(args, '-parallel-methods')
        if threads is not None:
          args = remove_arg(args, '-parallel-threads', has_param=True)
          args += ['-parallel-threads', str(threads)]

        with binary_util.safe_args(batch, self.get_options()) as batch_tests:
          self.context.log.debug('CWD = {}'.format(workdir))
          self.context.log.debug('platform = {}'.format(platform))
          with environment_as(**dict(target_env_vars)):
            result += abs(self._spawn_and_wait(
              executor=SubprocessExecutor(distribution),
              distribution=distribution,
              classpath=complete_classpath,
              main=JUnitRun._MAIN,
              jvm_options=self.jvm_options + extra_jvm_options + list(target_jvm_options),
              args=args + batch_tests,
              workunit_factory=self.context.new_workunit,
              workunit_name='run',
              workunit_labels=[WorkUnitLabel.TEST],
              cwd=workdir,
              synthetic_jar_dir=self.workdir,
              create_synthetic_jar=self.synthetic_classpath,
            ))

          if result != 0 and self._fail_fast:
            break

    if result != 0:
      failed_targets_and_tests = self._get_failed_targets(tests_to_targets)
      failed_targets = sorted(failed_targets_and_tests, key=lambda target: target.address.spec)
      error_message_lines = []
      if self._failure_summary:
        for target in failed_targets:
          error_message_lines.append('\n{0}{1}'.format(' '*4, target.address.spec))
          for test in sorted(failed_targets_and_tests[target]):
            error_message_lines.append('{0}{1}'.format(' '*8, test))
      error_message_lines.append(
        '\njava {main} ... exited non-zero ({code}); {failed} failed {targets}.'
          .format(main=JUnitRun._MAIN, code=result, failed=len(failed_targets),
                  targets=pluralize(len(failed_targets), 'target'))
      )
      raise TestFailedTaskError('\n'.join(error_message_lines), failed_targets=list(failed_targets))
Exemplo n.º 43
0
    def test_synthetic_target_runtime_platform_lookup(self):
        init_subsystem(
            JvmPlatform,
            options={
                "jvm-platform": {
                    "platforms": {
                        "default-platform": {"target": "8"},
                        "default-runtime-platform": {"target": "8"},
                        "target-platform": {"target": "8"},
                        "target-runtime-platform": {"target": "8"},
                        "parent-target-platform": {"target": "8"},
                        "parent-target-runtime-platform": {"target": "8"},
                    },
                    "default_platform": "default-platform",
                    "default_runtime_platform": None,
                }
            },
        )

        just_platform = self.make_target(
            "//:parent-with-runtime-platform", HasRuntimePlatform, platform="parent-target-platform"
        )
        just_runtime_platform = self.make_target(
            "//:parent-with-platform",
            HasRuntimePlatform,
            runtime_platform="parent-target-runtime-platform",
        )

        synth_none = self.make_target(
            "//:without-platforms",
            HasRuntimePlatform,
            synthetic=True,
            derived_from=just_runtime_platform,
        )
        synth_just_platform = self.make_target(
            "//:with-platform",
            HasRuntimePlatform,
            synthetic=True,
            derived_from=just_runtime_platform,
            platform="target-platform",
        )
        synth_just_runtime = self.make_target(
            "//:with-runtime-platform",
            HasRuntimePlatform,
            synthetic=True,
            derived_from=just_runtime_platform,
            runtime_platform="target-runtime-platform",
        )
        synth_both = self.make_target(
            "//:with-platform-and-runtime-platform",
            HasRuntimePlatform,
            synthetic=True,
            derived_from=just_runtime_platform,
            platform="target-platform",
            runtime_platform="target-runtime-platform",
        )
        synth_just_platform_with_parent_same = self.make_target(
            "//:with-platform-and-platform-parent",
            HasRuntimePlatform,
            synthetic=True,
            derived_from=just_platform,
            platform="target-platform",
        )

        instance = JvmPlatform.global_instance()
        assert (
            instance.get_runtime_platform_for_target(synth_none).name
            == "parent-target-runtime-platform"
        )
        assert (
            instance.get_runtime_platform_for_target(synth_just_platform).name
            == "parent-target-runtime-platform"
        )
        assert (
            instance.get_runtime_platform_for_target(synth_just_runtime).name
            == "target-runtime-platform"
        )
        assert (
            instance.get_runtime_platform_for_target(synth_both).name == "target-runtime-platform"
        )
        assert (
            instance.get_runtime_platform_for_target(synth_just_platform_with_parent_same).name
            == "default-platform"
        )
Exemplo n.º 44
0
    def _run_tests(self, test_registry, output_dir, coverage=None):
        if coverage:
            extra_jvm_options = coverage.extra_jvm_options
            classpath_prepend = coverage.classpath_prepend
            classpath_append = coverage.classpath_append
        else:
            extra_jvm_options = []
            classpath_prepend = ()
            classpath_append = ()

        tests_by_properties = test_registry.index(
            lambda tgt: tgt.cwd if tgt.cwd is not None else self._working_dir,
            lambda tgt: tgt.test_platform,
            lambda tgt: tgt.payload.extra_jvm_options,
            lambda tgt: tgt.payload.extra_env_vars,
            lambda tgt: tgt.concurrency, lambda tgt: tgt.threads)

        # the below will be None if not set, and we'll default back to runtime_classpath
        classpath_product = self.context.products.get_data(
            'instrument_classpath')

        result = 0
        for properties, tests in tests_by_properties.items():
            (workdir, platform, target_jvm_options, target_env_vars,
             concurrency, threads) = properties
            for batch in self._partition(tests):
                # Batches of test classes will likely exist within the same targets: dedupe them.
                relevant_targets = {
                    test_registry.get_owning_target(t)
                    for t in batch
                }
                complete_classpath = OrderedSet()
                complete_classpath.update(classpath_prepend)
                complete_classpath.update(
                    JUnit.global_instance().runner_classpath(self.context))
                complete_classpath.update(
                    self.classpath(relevant_targets,
                                   classpath_product=classpath_product))
                complete_classpath.update(classpath_append)
                distribution = JvmPlatform.preferred_jvm_distribution(
                    [platform], self._strict_jvm_version)

                # Override cmdline args with values from junit_test() target that specify concurrency:
                args = self._args(output_dir) + [u'-xmlreport']

                if concurrency is not None:
                    args = remove_arg(args, '-default-parallel')
                    if concurrency == JUnitTests.CONCURRENCY_SERIAL:
                        args = ensure_arg(args,
                                          '-default-concurrency',
                                          param='SERIAL')
                    elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES:
                        args = ensure_arg(args,
                                          '-default-concurrency',
                                          param='PARALLEL_CLASSES')
                    elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_METHODS:
                        args = ensure_arg(args,
                                          '-default-concurrency',
                                          param='PARALLEL_METHODS')
                    elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES_AND_METHODS:
                        args = ensure_arg(args,
                                          '-default-concurrency',
                                          param='PARALLEL_CLASSES_AND_METHODS')

                if threads is not None:
                    args = remove_arg(args,
                                      '-parallel-threads',
                                      has_param=True)
                    args += ['-parallel-threads', str(threads)]

                batch_test_specs = [test.render_test_spec() for test in batch]
                with argfile.safe_args(batch_test_specs,
                                       self.get_options()) as batch_tests:
                    self.context.log.debug('CWD = {}'.format(workdir))
                    self.context.log.debug('platform = {}'.format(platform))
                    with environment_as(**dict(target_env_vars)):
                        result += abs(
                            self._spawn_and_wait(
                                executor=SubprocessExecutor(distribution),
                                distribution=distribution,
                                classpath=complete_classpath,
                                main=JUnit.RUNNER_MAIN,
                                jvm_options=self.jvm_options +
                                extra_jvm_options + list(target_jvm_options),
                                args=args + batch_tests,
                                workunit_factory=self.context.new_workunit,
                                workunit_name='run',
                                workunit_labels=[WorkUnitLabel.TEST],
                                cwd=workdir,
                                synthetic_jar_dir=output_dir,
                                create_synthetic_jar=self.synthetic_classpath,
                            ))

                    if result != 0 and self._fail_fast:
                        break

        if result != 0:

            def error_handler(parse_error):
                # Just log and move on since the result is only used to characterize failures, and raising
                # an error here would just distract from the underlying test failures.
                self.context.log.error(
                    'Error parsing test result file {path}: {cause}'.format(
                        path=parse_error.junit_xml_path,
                        cause=parse_error.cause))

            target_to_failed_test = parse_failed_targets(
                test_registry, output_dir, error_handler)
            failed_targets = sorted(target_to_failed_test,
                                    key=lambda t: t.address.spec)
            error_message_lines = []
            if self._failure_summary:
                for target in failed_targets:
                    error_message_lines.append('\n{indent}{address}'.format(
                        indent=' ' * 4, address=target.address.spec))
                    for test in sorted(target_to_failed_test[target]):
                        error_message_lines.append(
                            '{indent}{classname}#{methodname}'.format(
                                indent=' ' * 8,
                                classname=test.classname,
                                methodname=test.methodname))
            error_message_lines.append(
                '\njava {main} ... exited non-zero ({code}); {failed} failed {targets}.'
                .format(main=JUnit.RUNNER_MAIN,
                        code=result,
                        failed=len(failed_targets),
                        targets=pluralize(len(failed_targets), 'target')))
            raise ErrorWhileTesting('\n'.join(error_message_lines),
                                    failed_targets=list(failed_targets))
Exemplo n.º 45
0
 def platform(self):
   return JvmPlatform.global_instance().get_platform_for_target(self)
Exemplo n.º 46
0
    def work_for_vts_rsc(vts, ctx):
      # Double check the cache before beginning compilation
      hit_cache = self.check_cache(vts, counter)
      target = ctx.target

      if not hit_cache:
        cp_entries = []

        # Include the current machine's jdk lib jars. This'll blow up remotely.
        # We need a solution for that.
        # Probably something to do with https://github.com/pantsbuild/pants/pull/6346
        distribution = JvmPlatform.preferred_jvm_distribution([ctx.target.platform], strict=True)
        jvm_lib_jars_abs = distribution.find_libs(['rt.jar', 'dt.jar', 'jce.jar', 'tools.jar'])
        cp_entries.extend(jvm_lib_jars_abs)

        classpath_abs = self._zinc.compile_classpath(
          'rsc_classpath',
          ctx.target,
          extra_cp_entries=self._extra_compile_time_classpath)

        jar_deps = [t for t in DependencyContext.global_instance().dependencies_respecting_strict_deps(target)
                    if isinstance(t, JarLibrary)]
        metacp_jar_classpath_abs = [y[1] for y in self._metacp_jars_classpath_product.get_for_targets(
          jar_deps
        )]
        jar_jar_paths = {y[1] for y in self.context.products.get_data('rsc_classpath').get_for_targets(jar_deps)}

        classpath_abs = [c for c in classpath_abs if c not in jar_jar_paths]


        classpath_rel = fast_relpath_collection(classpath_abs)
        metacp_jar_classpath_rel = fast_relpath_collection(metacp_jar_classpath_abs)
        cp_entries.extend(classpath_rel)

        ctx.ensure_output_dirs_exist()

        counter_val = str(counter()).rjust(counter.format_length(), b' ')
        counter_str = '[{}/{}] '.format(counter_val, counter.size)
        self.context.log.info(
          counter_str,
          'Rsc-ing ',
          items_to_report_element(ctx.sources, '{} source'.format(self.name())),
          ' in ',
          items_to_report_element([t.address.reference() for t in vts.targets], 'target'),
          ' (',
          ctx.target.address.spec,
          ').')

        tgt, = vts.targets
        with Timer() as timer:
          # Step 1: Convert classpath to SemanticDB
          # ---------------------------------------
          scalac_classpath_path_entries_abs = self.tool_classpath('workaround-metacp-dependency-classpath')
          scalac_classpath_path_entries = fast_relpath_collection(scalac_classpath_path_entries_abs)
          rsc_index_dir = fast_relpath(ctx.rsc_index_dir, get_buildroot())
          args = [
            '--verbose',
            # NB: Without this setting, rsc will be missing some symbols
            #     from the scala library.
            '--include-scala-library-synthetics', # TODO generate these once and cache them
            # NB: We need to add these extra dependencies in order to be able
            #     to find symbols used by the scalac jars.
            '--dependency-classpath', os.pathsep.join(scalac_classpath_path_entries + list(jar_jar_paths)),
            # NB: The directory to dump the semanticdb jars generated by metacp.
            '--out', rsc_index_dir,
            os.pathsep.join(cp_entries),
          ]
          metacp_wu = self._runtool(
            'scala.meta.cli.Metacp',
            'metacp',
            args,
            distribution,
            tgt=tgt,
            input_files=(scalac_classpath_path_entries + classpath_rel),
            output_dir=rsc_index_dir)
          metacp_stdout = stdout_contents(metacp_wu)
          metacp_result = json.loads(metacp_stdout)


          metai_classpath = self._collect_metai_classpath(
            metacp_result, classpath_rel, jvm_lib_jars_abs)

          # Step 1.5: metai Index the semanticdbs
          # -------------------------------------
          self._run_metai_tool(distribution, metai_classpath, rsc_index_dir, tgt)

          # Step 2: Outline Scala sources into SemanticDB
          # ---------------------------------------------
          rsc_outline_dir = fast_relpath(ctx.rsc_outline_dir, get_buildroot())
          rsc_out = os.path.join(rsc_outline_dir, 'META-INF/semanticdb/out.semanticdb')
          safe_mkdir(os.path.join(rsc_outline_dir, 'META-INF/semanticdb'))
          target_sources = ctx.sources
          args = [
            '-cp', os.pathsep.join(metai_classpath + metacp_jar_classpath_rel),
            '-out', rsc_out,
          ] + target_sources
          self._runtool(
            'rsc.cli.Main',
            'rsc',
            args,
            distribution,
            tgt=tgt,
            # TODO pass the input files from the target snapshot instead of the below
            # input_snapshot = ctx.target.sources_snapshot(scheduler=self.context._scheduler)
            input_files=target_sources + metai_classpath + metacp_jar_classpath_rel,
            output_dir=rsc_outline_dir)
          rsc_classpath = [rsc_outline_dir]

          # Step 2.5: Postprocess the rsc outputs
          # TODO: This is only necessary as a workaround for https://github.com/twitter/rsc/issues/199.
          # Ideally, Rsc would do this on its own.
          self._run_metai_tool(distribution,
            rsc_classpath,
            rsc_outline_dir,
            tgt,
            extra_input_files=(rsc_out,))


          # Step 3: Convert SemanticDB into an mjar
          # ---------------------------------------
          rsc_mjar_file = fast_relpath(ctx.rsc_mjar_file, get_buildroot())
          args = [
            '-out', rsc_mjar_file,
            os.pathsep.join(rsc_classpath),
          ]
          self._runtool(
            'scala.meta.cli.Mjar',
            'mjar',
            args,
            distribution,
            tgt=tgt,
            input_files=(
              rsc_out,
            ),
            output_dir=os.path.dirname(rsc_mjar_file)
            )
          self.context.products.get_data('rsc_classpath').add_for_target(
            ctx.target,
            [(conf, ctx.rsc_mjar_file) for conf in self._confs],
          )

        self._record_target_stats(tgt,
                                  len(cp_entries),
                                  len(target_sources),
                                  timer.elapsed,
                                  False,
                                  'rsc'
                                  )
        # Write any additional resources for this target to the target workdir.
        self.write_extra_resources(ctx)

      # Update the products with the latest classes.
      self.register_extra_products_from_contexts([ctx.target], compile_contexts)
Exemplo n.º 47
0
 def test_platform(self):
   if self.payload.test_platform:
     return JvmPlatform.global_instance().get_platform_by_name(self.payload.test_platform)
   return self.platform
Exemplo n.º 48
0
    def compile(self, ctx, args, dependency_classpath, upstream_analysis,
                settings, compiler_option_sets, zinc_file_manager,
                javac_plugin_map, scalac_plugin_map):
        classpath = (ctx.classes_dir.path, ) + tuple(
            ce.path for ce in dependency_classpath)

        if self.get_options().capture_classpath:
            self._record_compile_classpath(classpath, ctx.target,
                                           ctx.classes_dir.path)

        try:
            distribution = JvmPlatform.preferred_jvm_distribution([settings],
                                                                  strict=True)
        except DistributionLocator.Error:
            distribution = JvmPlatform.preferred_jvm_distribution([settings],
                                                                  strict=False)

        javac_cmd = ['{}/bin/javac'.format(distribution.real_home)]

        javac_cmd.extend([
            '-classpath',
            ':'.join(classpath),
        ])

        if settings.args:
            settings_args = settings.args
            if any('$JAVA_HOME' in a for a in settings.args):
                logger.debug(
                    'Substituting "$JAVA_HOME" with "{}" in jvm-platform args.'
                    .format(distribution.home))
                settings_args = (a.replace('$JAVA_HOME', distribution.home)
                                 for a in settings.args)
            javac_cmd.extend(settings_args)

            javac_cmd.extend([
                # TODO: support -release
                '-source',
                str(settings.source_level),
                '-target',
                str(settings.target_level),
            ])

        if self.execution_strategy == self.HERMETIC:
            javac_cmd.extend([
                # We need to strip the source root from our output files. Outputting to a directory, and
                # capturing that directory, does the job.
                # Unfortunately, javac errors if the directory you pass to -d doesn't exist, and we don't
                # have a convenient way of making a directory in the output tree, so let's just use the
                # working directory as our output dir.
                # This also has the benefit of not needing to strip leading directories from the returned
                # snapshot.
                '-d',
                '.',
            ])
        else:
            javac_cmd.extend([
                '-d',
                ctx.classes_dir.path,
            ])

        javac_cmd.extend(self._javac_plugin_args(javac_plugin_map))

        javac_cmd.extend(args)

        compiler_option_sets_args = self.get_merged_args_for_compiler_option_sets(
            compiler_option_sets)
        javac_cmd.extend(compiler_option_sets_args)

        with argfile.safe_args(ctx.sources,
                               self.get_options()) as batched_sources:
            javac_cmd.extend(batched_sources)

            if self.execution_strategy == self.HERMETIC:
                self._execute_hermetic_compile(javac_cmd, ctx)
            else:
                with self.context.new_workunit(name='javac',
                                               cmd=' '.join(javac_cmd),
                                               labels=[WorkUnitLabel.COMPILER
                                                       ]) as workunit:
                    self.context.log.debug('Executing {}'.format(
                        ' '.join(javac_cmd)))
                    p = subprocess.Popen(javac_cmd,
                                         stdout=workunit.output('stdout'),
                                         stderr=workunit.output('stderr'))
                    return_code = p.wait()
                    workunit.set_outcome(
                        WorkUnit.FAILURE if return_code else WorkUnit.SUCCESS)
                    if return_code:
                        raise TaskError(
                            'javac exited with return code {rc}'.format(
                                rc=return_code))
Exemplo n.º 49
0
 def execute(self):
     if JvmPlatform.global_instance().get_options().compiler == 'zinc':
         return super(ZincCompile, self).execute()
Exemplo n.º 50
0
    def _run_tests(self, tests_to_targets):
        if self._coverage:
            extra_jvm_options = self._coverage.extra_jvm_options
            classpath_prepend = self._coverage.classpath_prepend
            classpath_append = self._coverage.classpath_append
        else:
            extra_jvm_options = []
            classpath_prepend = ()
            classpath_append = ()

        tests_by_properties = self._tests_by_properties(
            tests_to_targets,
            self._infer_workdir,
            lambda target: target.test_platform,
            lambda target: target.payload.extra_jvm_options,
            lambda target: target.payload.extra_env_vars,
        )

        # the below will be None if not set, and we'll default back to runtime_classpath
        classpath_product = self.context.products.get_data("instrument_classpath")

        result = 0
        for (workdir, platform, target_jvm_options, target_env_vars), tests in tests_by_properties.items():
            for batch in self._partition(tests):
                # Batches of test classes will likely exist within the same targets: dedupe them.
                relevant_targets = set(map(tests_to_targets.get, batch))
                complete_classpath = OrderedSet()
                complete_classpath.update(classpath_prepend)
                complete_classpath.update(self.tool_classpath("junit"))
                complete_classpath.update(self.classpath(relevant_targets, classpath_product=classpath_product))
                complete_classpath.update(classpath_append)
                distribution = JvmPlatform.preferred_jvm_distribution([platform], self._strict_jvm_version)
                with binary_util.safe_args(batch, self.get_options()) as batch_tests:
                    self.context.log.debug("CWD = {}".format(workdir))
                    self.context.log.debug("platform = {}".format(platform))
                    with environment_as(**dict(target_env_vars)):
                        result += abs(
                            self._spawn_and_wait(
                                executor=SubprocessExecutor(distribution),
                                distribution=distribution,
                                classpath=complete_classpath,
                                main=JUnitRun._MAIN,
                                jvm_options=self.jvm_options + extra_jvm_options + list(target_jvm_options),
                                args=self._args + batch_tests + ["-xmlreport"],
                                workunit_factory=self.context.new_workunit,
                                workunit_name="run",
                                workunit_labels=[WorkUnitLabel.TEST],
                                cwd=workdir,
                                synthetic_jar_dir=self.workdir,
                                create_synthetic_jar=self.synthetic_classpath,
                            )
                        )

                    if result != 0 and self._fail_fast:
                        break

        if result != 0:
            failed_targets_and_tests = self._get_failed_targets(tests_to_targets)
            failed_targets = sorted(failed_targets_and_tests, key=lambda target: target.address.spec)
            error_message_lines = []
            if self._failure_summary:
                for target in failed_targets:
                    error_message_lines.append("\n{0}{1}".format(" " * 4, target.address.spec))
                    for test in sorted(failed_targets_and_tests[target]):
                        error_message_lines.append("{0}{1}".format(" " * 8, test))
            error_message_lines.append(
                "\njava {main} ... exited non-zero ({code}); {failed} failed {targets}.".format(
                    main=JUnitRun._MAIN,
                    code=result,
                    failed=len(failed_targets),
                    targets=pluralize(len(failed_targets), "target"),
                )
            )
            raise TestFailedTaskError("\n".join(error_message_lines), failed_targets=list(failed_targets))