def _local_jvm_distribution(settings=None): settings_args = [settings] if settings else [] try: local_distribution = JvmPlatform.preferred_jvm_distribution(settings_args, strict=True) except DistributionLocator.Error: local_distribution = JvmPlatform.preferred_jvm_distribution(settings_args, strict=False) return local_distribution
def _get_zinc_arguments(settings): """Extracts and formats the zinc arguments given in the jvm platform settings. This is responsible for the symbol substitution which replaces $JAVA_HOME with the path to an appropriate jvm distribution. :param settings: The jvm platform settings from which to extract the arguments. :type settings: :class:`JvmPlatformSettings` """ zinc_args = [ '-C-source', '-C{}'.format(settings.source_level), '-C-target', '-C{}'.format(settings.target_level), ] if settings.args: settings_args = settings.args if any('$JAVA_HOME' in a for a in settings.args): try: distribution = JvmPlatform.preferred_jvm_distribution([settings], strict=True) except DistributionLocator.Error: distribution = JvmPlatform.preferred_jvm_distribution([settings], strict=False) logger.debug('Substituting "$JAVA_HOME" with "{}" in jvm-platform args.' .format(distribution.home)) settings_args = (a.replace('$JAVA_HOME', distribution.home) for a in settings.args) zinc_args.extend(settings_args) return zinc_args
def compile(self, ctx, args, classpath, upstream_analysis, settings, fatal_warnings, zinc_file_manager, javac_plugin_map, scalac_plugin_map): try: distribution = JvmPlatform.preferred_jvm_distribution([settings], strict=True) except DistributionLocator.Error: distribution = JvmPlatform.preferred_jvm_distribution([settings], strict=False) javac_cmd = ['{}/bin/javac'.format(distribution.real_home)] javac_cmd.extend([ '-classpath', ':'.join(classpath), ]) if settings.args: settings_args = settings.args if any('$JAVA_HOME' in a for a in settings.args): logger.debug('Substituting "$JAVA_HOME" with "{}" in jvm-platform args.' .format(distribution.home)) settings_args = (a.replace('$JAVA_HOME', distribution.home) for a in settings.args) javac_cmd.extend(settings_args) javac_cmd.extend([ '-d', ctx.classes_dir, # TODO: support -release '-source', str(settings.source_level), '-target', str(settings.target_level), ]) javac_cmd.extend(self._javac_plugin_args(javac_plugin_map)) javac_cmd.extend(args) if fatal_warnings: javac_cmd.extend(self.get_options().fatal_warnings_enabled_args) else: javac_cmd.extend(self.get_options().fatal_warnings_disabled_args) with argfile.safe_args(ctx.sources, self.get_options()) as batched_sources: javac_cmd.extend(batched_sources) with self.context.new_workunit(name='javac', cmd=' '.join(javac_cmd), labels=[WorkUnitLabel.COMPILER]) as workunit: self.context.log.debug('Executing {}'.format(' '.join(javac_cmd))) p = subprocess.Popen(javac_cmd, stdout=workunit.output('stdout'), stderr=workunit.output('stderr')) return_code = p.wait() workunit.set_outcome(WorkUnit.FAILURE if return_code else WorkUnit.SUCCESS) if return_code: raise TaskError('javac exited with return code {rc}'.format(rc=return_code))
def platform(self): """Platform associated with this target. :return: The jvm platform object. :rtype: JvmPlatformSettings """ return JvmPlatform.global_instance().get_platform_for_target(self)
def execute(self): if JvmPlatform.global_instance().get_options().compiler != self.compiler_name: # If the requested compiler is not the one supported by this task, # bail early. return # In case we have no relevant targets and return early, create the requested product maps. self.create_empty_extra_products() relevant_targets = list(self.context.targets(predicate=self.select)) if not relevant_targets: return # Clone the compile_classpath to the runtime_classpath. classpath_product = self.create_runtime_classpath() fingerprint_strategy = DependencyContext.global_instance().create_fingerprint_strategy( classpath_product) # Note, JVM targets are validated (`vts.update()`) as they succeed. As a result, # we begin writing artifacts out to the cache immediately instead of waiting for # all targets to finish. with self.invalidated(relevant_targets, invalidate_dependents=True, fingerprint_strategy=fingerprint_strategy, topological_order=True) as invalidation_check: compile_contexts = {vt.target: self.create_compile_context(vt.target, vt.results_dir) for vt in invalidation_check.all_vts} self.do_compile( invalidation_check, compile_contexts, classpath_product, ) if not self.get_options().use_classpath_jars: # Once compilation has completed, replace the classpath entry for each target with # its jar'd representation. for ccs in compile_contexts.values(): cc = self.select_runtime_context(ccs) for conf in self._confs: classpath_product.remove_for_target(cc.target, [(conf, cc.classes_dir.path)]) classpath_product.add_for_target(cc.target, [(conf, cc.jar_file.path)])
def test_distribution_from_targets_passes_through_platforms(self): fake_dist = "a dist" java8_platform = JvmPlatformSettings('8', '8', []) targets = [ self.make_target('platformed_target', JvmTarget, platform='java8') ] with unittest.mock.patch.object( JvmPlatform, 'preferred_jvm_distribution') as plat_mock: with unittest.mock.patch.object(JvmPlatform.global_instance(), 'get_platform_for_target') as \ target_plat_mock: target_plat_mock.return_value = java8_platform plat_mock.return_value = fake_dist dist = self.task.preferred_jvm_distribution_for_targets( targets) plat_mock.assert_called_once_with([java8_platform], strict=False, jdk=False) self.assertEqual(fake_dist, dist)
def test_jvm_options_from_platform_shlexed(self): init_subsystem( JvmPlatform, options={ "jvm-platform": { "platforms": { "platform-with-shlexable-vm-options": { "target": "8", "jvm_options": ["-Dsomething -Dsomethingelse"], }, }, } }, ) instance = JvmPlatform.global_instance() need_shlex_options = instance.get_platform_by_name( "platform-with-shlexable-vm-options") assert ("-Dsomething", "-Dsomethingelse") == need_shlex_options.jvm_options
def test_distribution_from_targets_passes_through_platforms(self): fake_dist = "a dist" java8_platform = self.java8_platform() targets = [ self.make_target("platformed_target", JvmTarget, platform="java8") ] with unittest.mock.patch.object( JvmPlatform, "preferred_jvm_distribution") as plat_mock: with unittest.mock.patch.object( JvmPlatform.global_instance(), "get_platform_for_target") as target_plat_mock: target_plat_mock.return_value = java8_platform plat_mock.return_value = fake_dist dist = self.task.preferred_jvm_distribution_for_targets( targets) plat_mock.assert_called_once_with([java8_platform], strict=None, jdk=False) self.assertEqual(fake_dist, dist)
def test_jvm_options(self): init_subsystem( JvmPlatform, options={ "jvm-platform": { "platforms": { "platform-with-jvm-options": { "target": "8", "jvm_options": ["-Dsomething"], }, "platform-without-jvm-options": {"target": "8"}, }, } }, ) instance = JvmPlatform.global_instance() with_options = instance.get_platform_by_name("platform-with-jvm-options") without_options = instance.get_platform_by_name("platform-without-jvm-options") assert ("-Dsomething",) == with_options.jvm_options assert tuple() == without_options.jvm_options
def execute(self): if self.goal not in JvmPrepCommand.goals(): raise AssertionError('Got goal "{}". Expected goal to be one of {}'.format( self.goal, JvmPrepCommand.goals())) targets = self.context.targets(postorder=True, predicate=self.runnable_prep_cmd) compile_classpath = self.context.products.get_data('compile_classpath') classpath_products = self.context.products.get_data('runtime_classpath', compile_classpath.copy) with self.context.new_workunit(name='jvm_prep_command', labels=[WorkUnitLabel.PREP]) as workunit: for target in targets: distribution = JvmPlatform.preferred_jvm_distribution([target.platform]) executor = SubprocessExecutor(distribution) mainclass = target.payload.get_field_value('mainclass') args = target.payload.get_field_value('args', []) target_jvm_options = target.payload.get_field_value('jvm_options', []) cp = list(ClasspathUtil.classpath(target.closure(), classpath_products)) if not cp: raise TaskError('target {} has no classpath. (Add dependencies= parameter?' .format(target.address.spec)) self.context.log.info('Running prep command for {}'.format(target.address.spec)) returncode = distribution.execute_java( executor=executor, classpath=cp, main=mainclass, jvm_options=target_jvm_options, args=args, workunit_factory=self.context.new_workunit, workunit_name='run', workunit_labels=[WorkUnitLabel.PREP], ) workunit.set_outcome(WorkUnit.FAILURE if returncode else WorkUnit.SUCCESS) if returncode: raise TaskError('RunJvmPrepCommand failed to run {}'.format(mainclass))
def run_tests(self, fail_fast, test_targets, output_dir, coverage): test_registry = self._collect_test_targets(test_targets) if test_registry.empty: return TestResult.rc(0) coverage.instrument(output_dir) def parse_error_handler(parse_error): # Just log and move on since the result is only used to characterize failures, and raising # an error here would just distract from the underlying test failures. self.context.log.error( 'Error parsing test result file {path}: {cause}'.format( path=parse_error.xml_path, cause=parse_error.cause)) # The 'instrument_classpath' product below below will be `None` if not set, and we'll default # back to runtime_classpath classpath_product = self.context.products.get_data( 'instrument_classpath') result = 0 for batch_id, (properties, batch) in enumerate(self._iter_batches(test_registry)): (workdir, platform, target_jvm_options, target_env_vars, concurrency, threads) = properties batch_output_dir = output_dir if self._batched: batch_output_dir = os.path.join(batch_output_dir, 'batch-{}'.format(batch_id)) run_modifications = coverage.run_modifications(batch_output_dir) extra_jvm_options = run_modifications.extra_jvm_options # Batches of test classes will likely exist within the same targets: dedupe them. relevant_targets = { test_registry.get_owning_target(t) for t in batch } complete_classpath = OrderedSet() complete_classpath.update(run_modifications.classpath_prepend) complete_classpath.update(JUnit.global_instance().runner_classpath( self.context)) complete_classpath.update( self.classpath(relevant_targets, classpath_product=classpath_product)) distribution = JvmPlatform.preferred_jvm_distribution( [platform], self._strict_jvm_version) # Override cmdline args with values from junit_test() target that specify concurrency: args = self._args(fail_fast, batch_output_dir) + [u'-xmlreport'] if concurrency is not None: args = remove_arg(args, '-default-parallel') if concurrency == JUnitTests.CONCURRENCY_SERIAL: args = ensure_arg(args, '-default-concurrency', param='SERIAL') elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES: args = ensure_arg(args, '-default-concurrency', param='PARALLEL_CLASSES') elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_METHODS: args = ensure_arg(args, '-default-concurrency', param='PARALLEL_METHODS') elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES_AND_METHODS: args = ensure_arg(args, '-default-concurrency', param='PARALLEL_CLASSES_AND_METHODS') if threads is not None: args = remove_arg(args, '-parallel-threads', has_param=True) args += ['-parallel-threads', str(threads)] batch_test_specs = [test.render_test_spec() for test in batch] with argfile.safe_args(batch_test_specs, self.get_options()) as batch_tests: with self._chroot(relevant_targets, workdir) as chroot: self.context.log.debug('CWD = {}'.format(chroot)) self.context.log.debug('platform = {}'.format(platform)) with environment_as(**dict(target_env_vars)): subprocess_result = self._spawn_and_wait( executor=SubprocessExecutor(distribution), distribution=distribution, classpath=complete_classpath, main=JUnit.RUNNER_MAIN, jvm_options=self.jvm_options + extra_jvm_options + list(target_jvm_options), args=args + batch_tests, workunit_factory=self.context.new_workunit, workunit_name='run', workunit_labels=[WorkUnitLabel.TEST], cwd=chroot, synthetic_jar_dir=batch_output_dir, create_synthetic_jar=self.synthetic_classpath, ) self.context.log.debug( 'JUnit subprocess exited with result ({})'.format( subprocess_result)) result += abs(subprocess_result) tests_info = self.parse_test_info(batch_output_dir, parse_error_handler, ['classname']) for test_name, test_info in tests_info.items(): test_item = Test(test_info['classname'], test_name) test_target = test_registry.get_owning_target(test_item) self.report_all_info_for_single_test( self.options_scope, test_target, test_name, test_info) if result != 0 and fail_fast: break if result == 0: return TestResult.rc(0) target_to_failed_test = parse_failed_targets(test_registry, output_dir, parse_error_handler) def sort_owning_target(t): return t.address.spec if t else None failed_targets = sorted(target_to_failed_test, key=sort_owning_target) error_message_lines = [] if self._failure_summary: def render_owning_target(t): return t.address.reference() if t else '<Unknown Target>' for target in failed_targets: error_message_lines.append('\n{indent}{owner}'.format( indent=' ' * 4, owner=render_owning_target(target))) for test in sorted(target_to_failed_test[target]): error_message_lines.append( '{indent}{classname}#{methodname}'.format( indent=' ' * 8, classname=test.classname, methodname=test.methodname)) error_message_lines.append( '\njava {main} ... exited non-zero ({code}); {failed} failed {targets}.' .format(main=JUnit.RUNNER_MAIN, code=result, failed=len(failed_targets), targets=pluralize(len(failed_targets), 'target'))) return TestResult(msg='\n'.join(error_message_lines), rc=result, failed_targets=failed_targets)
def run_tests(self, fail_fast, test_targets, output_dir, coverage): test_registry = self._collect_test_targets(test_targets) if test_registry.empty: return TestResult.rc(0) coverage.instrument(output_dir) def parse_error_handler(parse_error): # Just log and move on since the result is only used to characterize failures, and raising # an error here would just distract from the underlying test failures. self.context.log.error('Error parsing test result file {path}: {cause}' .format(path=parse_error.xml_path, cause=parse_error.cause)) # The 'instrument_classpath' product below below will be `None` if not set, and we'll default # back to runtime_classpath classpath_product = self.context.products.get_data('instrument_classpath') result = 0 for batch_id, (properties, batch) in enumerate(self._iter_batches(test_registry)): (workdir, platform, target_jvm_options, target_env_vars, concurrency, threads) = properties batch_output_dir = output_dir if self._batched: batch_output_dir = os.path.join(batch_output_dir, 'batch-{}'.format(batch_id)) run_modifications = coverage.run_modifications(batch_output_dir) extra_jvm_options = run_modifications.extra_jvm_options # Batches of test classes will likely exist within the same targets: dedupe them. relevant_targets = {test_registry.get_owning_target(t) for t in batch} complete_classpath = OrderedSet() complete_classpath.update(run_modifications.classpath_prepend) complete_classpath.update(JUnit.global_instance().runner_classpath(self.context)) complete_classpath.update(self.classpath(relevant_targets, classpath_product=classpath_product)) distribution = JvmPlatform.preferred_jvm_distribution([platform], self._strict_jvm_version) # Override cmdline args with values from junit_test() target that specify concurrency: args = self._args(fail_fast, batch_output_dir) + [u'-xmlreport'] if concurrency is not None: args = remove_arg(args, '-default-parallel') if concurrency == JUnitTests.CONCURRENCY_SERIAL: args = ensure_arg(args, '-default-concurrency', param='SERIAL') elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES: args = ensure_arg(args, '-default-concurrency', param='PARALLEL_CLASSES') elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_METHODS: args = ensure_arg(args, '-default-concurrency', param='PARALLEL_METHODS') elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES_AND_METHODS: args = ensure_arg(args, '-default-concurrency', param='PARALLEL_CLASSES_AND_METHODS') if threads is not None: args = remove_arg(args, '-parallel-threads', has_param=True) args += ['-parallel-threads', str(threads)] batch_test_specs = [test.render_test_spec() for test in batch] with argfile.safe_args(batch_test_specs, self.get_options()) as batch_tests: with self._chroot(relevant_targets, workdir) as chroot: self.context.log.debug('CWD = {}'.format(chroot)) self.context.log.debug('platform = {}'.format(platform)) with environment_as(**dict(target_env_vars)): subprocess_result = self._spawn_and_wait( executor=SubprocessExecutor(distribution), distribution=distribution, classpath=complete_classpath, main=JUnit.RUNNER_MAIN, jvm_options=self.jvm_options + extra_jvm_options + list(target_jvm_options), args=args + batch_tests, workunit_factory=self.context.new_workunit, workunit_name='run', workunit_labels=[WorkUnitLabel.TEST], cwd=chroot, synthetic_jar_dir=batch_output_dir, create_synthetic_jar=self.synthetic_classpath, ) self.context.log.debug('JUnit subprocess exited with result ({})' .format(subprocess_result)) result += abs(subprocess_result) tests_info = self.parse_test_info(batch_output_dir, parse_error_handler, ['classname']) for test_name, test_info in tests_info.items(): test_item = Test(test_info['classname'], test_name) test_target = test_registry.get_owning_target(test_item) self.report_all_info_for_single_test(self.options_scope, test_target, test_name, test_info) if result != 0 and fail_fast: break if result == 0: return TestResult.rc(0) target_to_failed_test = parse_failed_targets(test_registry, output_dir, parse_error_handler) def sort_owning_target(t): return t.address.spec if t else None failed_targets = sorted(target_to_failed_test, key=sort_owning_target) error_message_lines = [] if self._failure_summary: def render_owning_target(t): return t.address.reference() if t else '<Unknown Target>' for target in failed_targets: error_message_lines.append('\n{indent}{owner}'.format(indent=' ' * 4, owner=render_owning_target(target))) for test in sorted(target_to_failed_test[target]): error_message_lines.append('{indent}{classname}#{methodname}' .format(indent=' ' * 8, classname=test.classname, methodname=test.methodname)) error_message_lines.append( '\njava {main} ... exited non-zero ({code}); {failed} failed {targets}.' .format(main=JUnit.RUNNER_MAIN, code=result, failed=len(failed_targets), targets=pluralize(len(failed_targets), 'target')) ) return TestResult(msg='\n'.join(error_message_lines), rc=result, failed_targets=failed_targets)
def platform(self): return JvmPlatform.global_instance().get_platform_for_target(self)
def console_output(self, targets): targets_map = {} resource_target_map = {} ivy_info = None if self.get_options().libraries: ivy_jar_products = self.context.products.get_data('ivy_jar_products') or {} # This product is a list for historical reasons (exclusives groups) but in practice should # have either 0 or 1 entries. ivy_info_list = ivy_jar_products.get('default') if ivy_info_list: assert len(ivy_info_list) == 1, ( 'The values in ivy_jar_products should always be length 1,' ' since we no longer have exclusives groups.' ) ivy_info = ivy_info_list[0] ivy_jar_memo = {} def process_target(current_target): """ :type current_target:pants.base.target.Target """ def get_target_type(target): if target.is_test: return Export.SourceRootTypes.TEST else: if (isinstance(target, Resources) and target in resource_target_map and resource_target_map[target].is_test): return Export.SourceRootTypes.TEST_RESOURCE elif isinstance(target, Resources): return Export.SourceRootTypes.RESOURCE else: return Export.SourceRootTypes.SOURCE def get_transitive_jars(jar_lib): """ :type jar_lib: pants.backend.jvm.targets.jar_library.JarLibrary :rtype: twitter.common.collections.orderedset.OrderedSet """ if not ivy_info or not self.get_options().libraries: return OrderedSet() transitive_jars = OrderedSet() for jar in jar_lib.jar_dependencies: transitive_jars.update(ivy_info.get_jars_for_ivy_module(jar, memo=ivy_jar_memo)) return transitive_jars info = { 'targets': [], 'libraries': [], 'roots': [], 'target_type': get_target_type(current_target), 'is_code_gen': current_target.is_codegen, 'pants_target_type': self._get_pants_target_alias(type(current_target)) } if not current_target.is_synthetic: info['globs'] = current_target.globs_relative_to_buildroot() if self.get_options().sources: info['sources'] = list(current_target.sources_relative_to_buildroot()) target_libraries = OrderedSet() if isinstance(current_target, JarLibrary): target_libraries = get_transitive_jars(current_target) for dep in current_target.dependencies: info['targets'].append(dep.address.spec) if isinstance(dep, JarLibrary): for jar in dep.jar_dependencies: target_libraries.add(IvyModuleRef(jar.org, jar.name, jar.rev)) # Add all the jars pulled in by this jar_library target_libraries.update(get_transitive_jars(dep)) if isinstance(dep, Resources): resource_target_map[dep] = current_target if isinstance(current_target, ScalaLibrary): for dep in current_target.java_sources: info['targets'].append(dep.address.spec) process_target(dep) if isinstance(current_target, JvmTarget): info['excludes'] = [self._exclude_id(exclude) for exclude in current_target.excludes] info['platform'] = current_target.platform.name info['roots'] = map(lambda (source_root, package_prefix): { 'source_root': source_root, 'package_prefix': package_prefix }, self._source_roots_for_target(current_target)) if self.get_options().libraries: info['libraries'] = [self._jar_id(lib) for lib in target_libraries] targets_map[current_target.address.spec] = info for target in targets: process_target(target) jvm_platforms_map = { 'default_platform' : JvmPlatform.global_instance().default_platform.name, 'platforms': { str(platform_name): { 'target_level' : str(platform.target_level), 'source_level' : str(platform.source_level), 'args' : platform.args, } for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items() } } graph_info = { 'targets': targets_map, 'jvm_platforms' : jvm_platforms_map, } jvm_distributions = DistributionLocator.global_instance().all_jdk_paths() if jvm_distributions: graph_info['jvm_distributions'] = jvm_distributions if self.get_options().libraries: graph_info['libraries'] = self._resolve_jars_info() graph_info['version'] = self.DEFAULT_EXPORT_VERSION if self.format: return json.dumps(graph_info, indent=4, separators=(',', ': ')).splitlines() else: return [json.dumps(graph_info)]
def console_output(self, targets): targets_map = {} resource_target_map = {} classpath_products = ( self.context.products.get_data("compile_classpath") if self.get_options().libraries else None ) python_interpreter_targets_mapping = defaultdict(list) def process_target(current_target): """ :type current_target:pants.build_graph.target.Target """ def get_target_type(target): if target.is_test: return Export.SourceRootTypes.TEST else: if ( isinstance(target, Resources) and target in resource_target_map and resource_target_map[target].is_test ): return Export.SourceRootTypes.TEST_RESOURCE elif isinstance(target, Resources): return Export.SourceRootTypes.RESOURCE else: return Export.SourceRootTypes.SOURCE info = { "targets": [], "libraries": [], "roots": [], "target_type": get_target_type(current_target), "is_code_gen": current_target.is_codegen, "pants_target_type": self._get_pants_target_alias(type(current_target)), } if not current_target.is_synthetic: info["globs"] = current_target.globs_relative_to_buildroot() if self.get_options().sources: info["sources"] = list(current_target.sources_relative_to_buildroot()) if isinstance(current_target, PythonRequirementLibrary): reqs = current_target.payload.get_field_value("requirements", set()) """:type : set[pants.backend.python.python_requirement.PythonRequirement]""" info["requirements"] = [req.key for req in reqs] if isinstance(current_target, PythonTarget): interpreter_for_target = self.select_interpreter_for_targets([current_target]) if interpreter_for_target is None: raise TaskError("Unable to find suitable interpreter for {}".format(current_target.address)) python_interpreter_targets_mapping[interpreter_for_target].append(current_target) info["python_interpreter"] = str(interpreter_for_target.identity) def iter_transitive_jars(jar_lib): """ :type jar_lib: :class:`pants.backend.jvm.targets.jar_library.JarLibrary` :rtype: :class:`collections.Iterator` of :class:`pants.backend.jvm.jar_dependency_utils.M2Coordinate` """ if classpath_products: jar_products = classpath_products.get_artifact_classpath_entries_for_targets((jar_lib,)) for _, jar_entry in jar_products: coordinate = jar_entry.coordinate # We drop classifier and type_ since those fields are represented in the global # libraries dict and here we just want the key into that dict (see `_jar_id`). yield M2Coordinate(org=coordinate.org, name=coordinate.name, rev=coordinate.rev) target_libraries = OrderedSet() if isinstance(current_target, JarLibrary): target_libraries = OrderedSet(iter_transitive_jars(current_target)) for dep in current_target.dependencies: info["targets"].append(dep.address.spec) if isinstance(dep, JarLibrary): for jar in dep.jar_dependencies: target_libraries.add(M2Coordinate(jar.org, jar.name, jar.rev)) # Add all the jars pulled in by this jar_library target_libraries.update(iter_transitive_jars(dep)) if isinstance(dep, Resources): resource_target_map[dep] = current_target if isinstance(current_target, ScalaLibrary): for dep in current_target.java_sources: info["targets"].append(dep.address.spec) process_target(dep) if isinstance(current_target, JvmTarget): info["excludes"] = [self._exclude_id(exclude) for exclude in current_target.excludes] info["platform"] = current_target.platform.name info["roots"] = map( lambda (source_root, package_prefix): {"source_root": source_root, "package_prefix": package_prefix}, self._source_roots_for_target(current_target), ) if classpath_products: info["libraries"] = [self._jar_id(lib) for lib in target_libraries] targets_map[current_target.address.spec] = info for target in targets: process_target(target) jvm_platforms_map = { "default_platform": JvmPlatform.global_instance().default_platform.name, "platforms": { str(platform_name): { "target_level": str(platform.target_level), "source_level": str(platform.source_level), "args": platform.args, } for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items() }, } graph_info = { "version": self.DEFAULT_EXPORT_VERSION, "targets": targets_map, "jvm_platforms": jvm_platforms_map, } jvm_distributions = DistributionLocator.global_instance().all_jdk_paths() if jvm_distributions: graph_info["jvm_distributions"] = jvm_distributions if classpath_products: graph_info["libraries"] = self._resolve_jars_info(targets, classpath_products) if python_interpreter_targets_mapping: interpreters = self.interpreter_cache.select_interpreter(python_interpreter_targets_mapping.keys()) default_interpreter = interpreters[0] interpreters_info = {} for interpreter, targets in six.iteritems(python_interpreter_targets_mapping): chroot = self.cached_chroot(interpreter=interpreter, pex_info=PexInfo.default(), targets=targets) interpreters_info[str(interpreter.identity)] = {"binary": interpreter.binary, "chroot": chroot.path()} graph_info["python_setup"] = { "default_interpreter": str(default_interpreter.identity), "interpreters": interpreters_info, } if self.format: return json.dumps(graph_info, indent=4, separators=(",", ": ")).splitlines() else: return [json.dumps(graph_info)]
def work_for_vts_rsc_jar_library(vts, ctx): cp_entries = [] # Include the current machine's jdk lib jars. This'll blow up remotely. # We need a solution for that. # Probably something to do with https://github.com/pantsbuild/pants/pull/6346 # TODO perhaps determine the platform of the jar and use that here. # https://github.com/pantsbuild/pants/issues/6547 distribution = JvmPlatform.preferred_jvm_distribution([], strict=True) jvm_lib_jars_abs = distribution.find_libs(['rt.jar', 'dt.jar', 'jce.jar', 'tools.jar']) cp_entries.extend(jvm_lib_jars_abs) # TODO use compile_classpath classpath_abs = [ path for (conf, path) in self.context.products.get_data('rsc_classpath').get_for_target(ctx.target) ] dependency_classpath = self._zinc.compile_classpath( 'compile_classpath', ctx.target, extra_cp_entries=self._extra_compile_time_classpath) classpath_rel = fast_relpath_collection(classpath_abs) cp_entries.extend(classpath_rel) counter_val = str(counter()).rjust(counter.format_length(), b' ') counter_str = '[{}/{}] '.format(counter_val, counter.size) self.context.log.info( counter_str, 'Metacp-ing ', items_to_report_element(cp_entries, 'jar'), ' in ', items_to_report_element([t.address.reference() for t in vts.targets], 'target'), ' (', ctx.target.address.spec, ').') ctx.ensure_output_dirs_exist() tgt, = vts.targets with Timer() as timer: # Step 1: Convert classpath to SemanticDB # --------------------------------------- scalac_classpath_path_entries_abs = self.tool_classpath('workaround-metacp-dependency-classpath') scalac_classpath_path_entries = fast_relpath_collection(scalac_classpath_path_entries_abs) rsc_index_dir = fast_relpath(ctx.rsc_index_dir, get_buildroot()) args = [ '--verbose', # NB: Without this setting, rsc will be missing some symbols # from the scala library. '--include-scala-library-synthetics', # TODO generate these once and cache them # NB: We need to add these extra dependencies in order to be able # to find symbols used by the scalac jars. '--dependency-classpath', os.pathsep.join(dependency_classpath + scalac_classpath_path_entries), # NB: The directory to dump the semanticdb jars generated by metacp. '--out', rsc_index_dir, os.pathsep.join(cp_entries), ] metacp_wu = self._runtool( 'scala.meta.cli.Metacp', 'metacp', args, distribution, tgt=tgt, input_files=(scalac_classpath_path_entries + classpath_rel), output_dir=rsc_index_dir) metacp_stdout = stdout_contents(metacp_wu) metacp_result = json.loads(metacp_stdout) metai_classpath = self._collect_metai_classpath( metacp_result, classpath_rel, jvm_lib_jars_abs) # Step 1.5: metai Index the semanticdbs # ------------------------------------- self._run_metai_tool(distribution, metai_classpath, rsc_index_dir, tgt) abs_output = [(conf, os.path.join(get_buildroot(), x)) for conf in self._confs for x in metai_classpath] self._metacp_jars_classpath_product.add_for_target( ctx.target, abs_output, ) self._record_target_stats(tgt, len(abs_output), len([]), timer.elapsed, False, 'metacp' )
def generate_targets_map(self, targets, classpath_products=None): """Generates a dictionary containing all pertinent information about the target graph. The return dictionary is suitable for serialization by json.dumps. :param targets: The list of targets to generate the map for. :param classpath_products: Optional classpath_products. If not provided when the --libraries option is `True`, this task will perform its own jar resolution. """ targets_map = {} resource_target_map = {} python_interpreter_targets_mapping = defaultdict(list) if self.get_options().libraries: # NB(gmalmquist): This supports mocking the classpath_products in tests. if classpath_products is None: classpath_products = self.resolve_jars(targets) else: classpath_products = None target_roots_set = set(self.context.target_roots) def process_target(current_target): """ :type current_target:pants.build_graph.target.Target """ def get_target_type(tgt): def is_test(t): return isinstance(t, JUnitTests) or isinstance( t, PythonTests) if is_test(tgt): return SourceRootTypes.TEST else: if (isinstance(tgt, Resources) and tgt in resource_target_map and is_test(resource_target_map[tgt])): return SourceRootTypes.TEST_RESOURCE elif isinstance(tgt, Resources): return SourceRootTypes.RESOURCE else: return SourceRootTypes.SOURCE info = { 'targets': [], 'libraries': [], 'roots': [], 'id': current_target.id, 'target_type': get_target_type(current_target), # NB: is_code_gen should be removed when export format advances to 1.1.0 or higher 'is_code_gen': current_target.is_synthetic, 'is_synthetic': current_target.is_synthetic, 'pants_target_type': self._get_pants_target_alias(type(current_target)), } if not current_target.is_synthetic: info['globs'] = current_target.globs_relative_to_buildroot() if self.get_options().sources: info['sources'] = list( current_target.sources_relative_to_buildroot()) info['transitive'] = current_target.transitive info['scope'] = str(current_target.scope) info['is_target_root'] = current_target in target_roots_set if isinstance(current_target, PythonRequirementLibrary): reqs = current_target.payload.get_field_value( 'requirements', set()) """:type : set[pants.backend.python.python_requirement.PythonRequirement]""" info['requirements'] = [req.key for req in reqs] if isinstance(current_target, PythonTarget): interpreter_for_target = self._interpreter_cache.select_interpreter_for_targets( [current_target]) if interpreter_for_target is None: raise TaskError( 'Unable to find suitable interpreter for {}'.format( current_target.address)) python_interpreter_targets_mapping[ interpreter_for_target].append(current_target) info['python_interpreter'] = str( interpreter_for_target.identity) def iter_transitive_jars(jar_lib): """ :type jar_lib: :class:`pants.backend.jvm.targets.jar_library.JarLibrary` :rtype: :class:`collections.Iterator` of :class:`pants.java.jar.M2Coordinate` """ if classpath_products: jar_products = classpath_products.get_artifact_classpath_entries_for_targets( (jar_lib, )) for _, jar_entry in jar_products: coordinate = jar_entry.coordinate # We drop classifier and type_ since those fields are represented in the global # libraries dict and here we just want the key into that dict (see `_jar_id`). yield M2Coordinate(org=coordinate.org, name=coordinate.name, rev=coordinate.rev) target_libraries = OrderedSet() if isinstance(current_target, JarLibrary): target_libraries = OrderedSet( iter_transitive_jars(current_target)) for dep in current_target.dependencies: info['targets'].append(dep.address.spec) if isinstance(dep, JarLibrary): for jar in dep.jar_dependencies: target_libraries.add( M2Coordinate(jar.org, jar.name, jar.rev)) # Add all the jars pulled in by this jar_library target_libraries.update(iter_transitive_jars(dep)) if isinstance(dep, Resources): resource_target_map[dep] = current_target if isinstance(current_target, ScalaLibrary): for dep in current_target.java_sources: info['targets'].append(dep.address.spec) process_target(dep) if isinstance(current_target, JvmTarget): info['excludes'] = [ self._exclude_id(exclude) for exclude in current_target.excludes ] info['platform'] = current_target.platform.name if hasattr(current_target, 'test_platform'): info['test_platform'] = current_target.test_platform.name info['roots'] = [{ 'source_root': source_root_package_prefix[0], 'package_prefix': source_root_package_prefix[1] } for source_root_package_prefix in self._source_roots_for_target( current_target)] if classpath_products: info['libraries'] = [ self._jar_id(lib) for lib in target_libraries ] targets_map[current_target.address.spec] = info for target in targets: process_target(target) scala_platform = ScalaPlatform.global_instance() scala_platform_map = { 'scala_version': scala_platform.version, 'compiler_classpath': [ cp_entry.path for cp_entry in scala_platform.compiler_classpath_entries( self.context.products) ], } jvm_platforms_map = { 'default_platform': JvmPlatform.global_instance().default_platform.name, 'platforms': { str(platform_name): { 'target_level': str(platform.target_level), 'source_level': str(platform.source_level), 'args': platform.args, } for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items() }, } graph_info = { 'version': DEFAULT_EXPORT_VERSION, 'targets': targets_map, 'jvm_platforms': jvm_platforms_map, 'scala_platform': scala_platform_map, # `jvm_distributions` are static distribution settings from config, # `preferred_jvm_distributions` are distributions that pants actually uses for the # given platform setting. 'preferred_jvm_distributions': {} } for platform_name, platform in JvmPlatform.global_instance( ).platforms_by_name.items(): preferred_distributions = {} for strict, strict_key in [(True, 'strict'), (False, 'non_strict')]: try: dist = JvmPlatform.preferred_jvm_distribution( [platform], strict=strict) preferred_distributions[strict_key] = dist.home except DistributionLocator.Error: pass if preferred_distributions: graph_info['preferred_jvm_distributions'][ platform_name] = preferred_distributions if classpath_products: graph_info['libraries'] = self._resolve_jars_info( targets, classpath_products) if python_interpreter_targets_mapping: # NB: We've selected a python interpreter compatible with each python target individually into # the `python_interpreter_targets_mapping`. These python targets may not be compatible, ie: we # could have a python target requiring 'CPython>=2.7<3' (ie: CPython-2.7.x) and another # requiring 'CPython>=3.6'. To pick a default interpreter then from among these two choices # is arbitrary and not to be relied on to work as a default interpreter if ever needed by the # export consumer. # # TODO(John Sirois): consider either eliminating the 'default_interpreter' field and pressing # export consumers to make their own choice of a default (if needed) or else use # `select.select_interpreter_for_targets` and fail fast if there is no interpreter compatible # across all the python targets in-play. # # For now, make our arbitrary historical choice of a default interpreter explicit and use the # lowest version. default_interpreter = min( python_interpreter_targets_mapping.keys()) interpreters_info = {} for interpreter, targets in python_interpreter_targets_mapping.items( ): req_libs = [ target for target in Target.closure_for_targets(targets) if has_python_requirements(target) ] chroot = self.resolve_requirements(interpreter, req_libs) interpreters_info[str(interpreter.identity)] = { 'binary': interpreter.binary, 'chroot': chroot.path() } graph_info['python_setup'] = { 'default_interpreter': str(default_interpreter.identity), 'interpreters': interpreters_info } if self.get_options().available_target_types: graph_info['available_target_types'] = self._target_types() return graph_info
def _run_tests(self, test_registry, output_dir, coverage=None): if coverage: extra_jvm_options = coverage.extra_jvm_options classpath_prepend = coverage.classpath_prepend classpath_append = coverage.classpath_append else: extra_jvm_options = [] classpath_prepend = () classpath_append = () tests_by_properties = test_registry.index( lambda tgt: tgt.cwd if tgt.cwd is not None else self._working_dir, lambda tgt: tgt.test_platform, lambda tgt: tgt.payload.extra_jvm_options, lambda tgt: tgt.payload.extra_env_vars, lambda tgt: tgt.concurrency, lambda tgt: tgt.threads) # the below will be None if not set, and we'll default back to runtime_classpath classpath_product = self.context.products.get_data('instrument_classpath') result = 0 for properties, tests in tests_by_properties.items(): (workdir, platform, target_jvm_options, target_env_vars, concurrency, threads) = properties for batch in self._partition(tests): # Batches of test classes will likely exist within the same targets: dedupe them. relevant_targets = {test_registry.get_owning_target(t) for t in batch} complete_classpath = OrderedSet() complete_classpath.update(classpath_prepend) complete_classpath.update(JUnit.global_instance().runner_classpath(self.context)) complete_classpath.update(self.classpath(relevant_targets, classpath_product=classpath_product)) complete_classpath.update(classpath_append) distribution = JvmPlatform.preferred_jvm_distribution([platform], self._strict_jvm_version) # Override cmdline args with values from junit_test() target that specify concurrency: args = self._args(output_dir) + [u'-xmlreport'] if concurrency is not None: args = remove_arg(args, '-default-parallel') if concurrency == JUnitTests.CONCURRENCY_SERIAL: args = ensure_arg(args, '-default-concurrency', param='SERIAL') elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES: args = ensure_arg(args, '-default-concurrency', param='PARALLEL_CLASSES') elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_METHODS: args = ensure_arg(args, '-default-concurrency', param='PARALLEL_METHODS') elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES_AND_METHODS: args = ensure_arg(args, '-default-concurrency', param='PARALLEL_CLASSES_AND_METHODS') if threads is not None: args = remove_arg(args, '-parallel-threads', has_param=True) args += ['-parallel-threads', str(threads)] batch_test_specs = [test.render_test_spec() for test in batch] with argfile.safe_args(batch_test_specs, self.get_options()) as batch_tests: self.context.log.debug('CWD = {}'.format(workdir)) self.context.log.debug('platform = {}'.format(platform)) with environment_as(**dict(target_env_vars)): result += abs(self._spawn_and_wait( executor=SubprocessExecutor(distribution), distribution=distribution, classpath=complete_classpath, main=JUnit.RUNNER_MAIN, jvm_options=self.jvm_options + extra_jvm_options + list(target_jvm_options), args=args + batch_tests, workunit_factory=self.context.new_workunit, workunit_name='run', workunit_labels=[WorkUnitLabel.TEST], cwd=workdir, synthetic_jar_dir=output_dir, create_synthetic_jar=self.synthetic_classpath, )) if result != 0 and self._fail_fast: break if result != 0: def error_handler(parse_error): # Just log and move on since the result is only used to characterize failures, and raising # an error here would just distract from the underlying test failures. self.context.log.error('Error parsing test result file {path}: {cause}' .format(path=parse_error.junit_xml_path, cause=parse_error.cause)) target_to_failed_test = parse_failed_targets(test_registry, output_dir, error_handler) failed_targets = sorted(target_to_failed_test, key=lambda t: t.address.spec) error_message_lines = [] if self._failure_summary: for target in failed_targets: error_message_lines.append('\n{indent}{address}'.format(indent=' ' * 4, address=target.address.spec)) for test in sorted(target_to_failed_test[target]): error_message_lines.append('{indent}{classname}#{methodname}' .format(indent=' ' * 8, classname=test.classname, methodname=test.methodname)) error_message_lines.append( '\njava {main} ... exited non-zero ({code}); {failed} failed {targets}.' .format(main=JUnit.RUNNER_MAIN, code=result, failed=len(failed_targets), targets=pluralize(len(failed_targets), 'target')) ) raise TestFailedTaskError('\n'.join(error_message_lines), failed_targets=list(failed_targets))
def compile(self, ctx, args, classpath, upstream_analysis, settings, fatal_warnings, zinc_file_manager, javac_plugin_map, scalac_plugin_map): try: distribution = JvmPlatform.preferred_jvm_distribution([settings], strict=True) except DistributionLocator.Error: distribution = JvmPlatform.preferred_jvm_distribution([settings], strict=False) javac_cmd = ['{}/bin/javac'.format(distribution.real_home)] javac_cmd.extend([ '-classpath', ':'.join(classpath), ]) if settings.args: settings_args = settings.args if any('$JAVA_HOME' in a for a in settings.args): logger.debug( 'Substituting "$JAVA_HOME" with "{}" in jvm-platform args.' .format(distribution.home)) settings_args = (a.replace('$JAVA_HOME', distribution.home) for a in settings.args) javac_cmd.extend(settings_args) javac_cmd.extend([ '-d', ctx.classes_dir, # TODO: support -release '-source', str(settings.source_level), '-target', str(settings.target_level), ]) javac_cmd.extend(self._javac_plugin_args(javac_plugin_map)) javac_cmd.extend(args) if fatal_warnings: javac_cmd.extend(self.get_options().fatal_warnings_enabled_args) else: javac_cmd.extend(self.get_options().fatal_warnings_disabled_args) with argfile.safe_args(ctx.sources, self.get_options()) as batched_sources: javac_cmd.extend(batched_sources) with self.context.new_workunit(name='javac', cmd=' '.join(javac_cmd), labels=[WorkUnitLabel.COMPILER ]) as workunit: self.context.log.debug('Executing {}'.format( ' '.join(javac_cmd))) p = subprocess.Popen(javac_cmd, stdout=workunit.output('stdout'), stderr=workunit.output('stderr')) return_code = p.wait() workunit.set_outcome( WorkUnit.FAILURE if return_code else WorkUnit.SUCCESS) if return_code: raise TaskError( 'javac exited with return code {rc}'.format( rc=return_code))
def _run_tests(self, tests_to_targets): if self._coverage: extra_jvm_options = self._coverage.extra_jvm_options classpath_prepend = self._coverage.classpath_prepend classpath_append = self._coverage.classpath_append else: extra_jvm_options = [] classpath_prepend = () classpath_append = () tests_by_properties = self._tests_by_properties( tests_to_targets, self._infer_workdir, lambda target: target.test_platform, lambda target: target.payload.extra_jvm_options, lambda target: target.payload.extra_env_vars, ) # the below will be None if not set, and we'll default back to runtime_classpath classpath_product = self.context.products.get_data('instrument_classpath') result = 0 for (workdir, platform, target_jvm_options, target_env_vars), tests in tests_by_properties.items(): for batch in self._partition(tests): # Batches of test classes will likely exist within the same targets: dedupe them. relevant_targets = set(map(tests_to_targets.get, batch)) complete_classpath = OrderedSet() complete_classpath.update(classpath_prepend) complete_classpath.update(self.tool_classpath('junit')) complete_classpath.update(self.classpath(relevant_targets, classpath_product=classpath_product)) complete_classpath.update(classpath_append) distribution = JvmPlatform.preferred_jvm_distribution([platform], self._strict_jvm_version) with binary_util.safe_args(batch, self.get_options()) as batch_tests: self.context.log.debug('CWD = {}'.format(workdir)) self.context.log.debug('platform = {}'.format(platform)) with environment_as(**dict(target_env_vars)): result += abs(self._spawn_and_wait( executor=SubprocessExecutor(distribution), distribution=distribution, classpath=complete_classpath, main=JUnitRun._MAIN, jvm_options=self.jvm_options + extra_jvm_options + list(target_jvm_options), args=self._args + batch_tests + [u'-xmlreport'], workunit_factory=self.context.new_workunit, workunit_name='run', workunit_labels=[WorkUnitLabel.TEST], cwd=workdir, synthetic_jar_dir=self.workdir, create_synthetic_jar=self.synthetic_classpath, )) if result != 0 and self._fail_fast: break if result != 0: failed_targets_and_tests = self._get_failed_targets(tests_to_targets) failed_targets = sorted(failed_targets_and_tests, key=lambda target: target.address.spec) error_message_lines = [] if self._failure_summary: for target in failed_targets: error_message_lines.append('\n{0}{1}'.format(' '*4, target.address.spec)) for test in sorted(failed_targets_and_tests[target]): error_message_lines.append('{0}{1}'.format(' '*8, test)) error_message_lines.append( '\njava {main} ... exited non-zero ({code}); {failed} failed {targets}.' .format(main=JUnitRun._MAIN, code=result, failed=len(failed_targets), targets=pluralize(len(failed_targets), 'target')) ) raise TestFailedTaskError('\n'.join(error_message_lines), failed_targets=list(failed_targets))
def generate_targets_map(self, targets, classpath_products=None): """Generates a dictionary containing all pertinent information about the target graph. The return dictionary is suitable for serialization by json.dumps. :param targets: The list of targets to generate the map for. :param classpath_products: Optional classpath_products. If not provided when the --libraries option is `True`, this task will perform its own jar resolution. """ targets_map = {} resource_target_map = {} python_interpreter_targets_mapping = defaultdict(list) if self.get_options().libraries: # NB(gmalmquist): This supports mocking the classpath_products in tests. if classpath_products is None: classpath_products = self.resolve_jars(targets) else: classpath_products = None target_roots_set = set(self.context.target_roots) def process_target(current_target): """ :type current_target:pants.build_graph.target.Target """ def get_target_type(tgt): def is_test(t): return isinstance(t, JUnitTests) or isinstance(t, PythonTests) if is_test(tgt): return ExportTask.SourceRootTypes.TEST else: if (isinstance(tgt, Resources) and tgt in resource_target_map and is_test(resource_target_map[tgt])): return ExportTask.SourceRootTypes.TEST_RESOURCE elif isinstance(tgt, Resources): return ExportTask.SourceRootTypes.RESOURCE else: return ExportTask.SourceRootTypes.SOURCE info = { 'targets': [], 'libraries': [], 'roots': [], 'id': current_target.id, 'target_type': get_target_type(current_target), # NB: is_code_gen should be removed when export format advances to 1.1.0 or higher 'is_code_gen': current_target.is_synthetic, 'is_synthetic': current_target.is_synthetic, 'pants_target_type': self._get_pants_target_alias(type(current_target)), } if not current_target.is_synthetic: info['globs'] = current_target.globs_relative_to_buildroot() if self.get_options().sources: info['sources'] = list(current_target.sources_relative_to_buildroot()) info['transitive'] = current_target.transitive info['scope'] = str(current_target.scope) info['is_target_root'] = current_target in target_roots_set if isinstance(current_target, PythonRequirementLibrary): reqs = current_target.payload.get_field_value('requirements', set()) """:type : set[pants.backend.python.python_requirement.PythonRequirement]""" info['requirements'] = [req.key for req in reqs] if isinstance(current_target, PythonTarget): interpreter_for_target = self._interpreter_cache.select_interpreter_for_targets( [current_target]) if interpreter_for_target is None: raise TaskError('Unable to find suitable interpreter for {}' .format(current_target.address)) python_interpreter_targets_mapping[interpreter_for_target].append(current_target) info['python_interpreter'] = str(interpreter_for_target.identity) def iter_transitive_jars(jar_lib): """ :type jar_lib: :class:`pants.backend.jvm.targets.jar_library.JarLibrary` :rtype: :class:`collections.Iterator` of :class:`pants.java.jar.M2Coordinate` """ if classpath_products: jar_products = classpath_products.get_artifact_classpath_entries_for_targets((jar_lib,)) for _, jar_entry in jar_products: coordinate = jar_entry.coordinate # We drop classifier and type_ since those fields are represented in the global # libraries dict and here we just want the key into that dict (see `_jar_id`). yield M2Coordinate(org=coordinate.org, name=coordinate.name, rev=coordinate.rev) target_libraries = OrderedSet() if isinstance(current_target, JarLibrary): target_libraries = OrderedSet(iter_transitive_jars(current_target)) for dep in current_target.dependencies: info['targets'].append(dep.address.spec) if isinstance(dep, JarLibrary): for jar in dep.jar_dependencies: target_libraries.add(M2Coordinate(jar.org, jar.name, jar.rev)) # Add all the jars pulled in by this jar_library target_libraries.update(iter_transitive_jars(dep)) if isinstance(dep, Resources): resource_target_map[dep] = current_target if isinstance(current_target, ScalaLibrary): for dep in current_target.java_sources: info['targets'].append(dep.address.spec) process_target(dep) if isinstance(current_target, JvmTarget): info['excludes'] = [self._exclude_id(exclude) for exclude in current_target.excludes] info['platform'] = current_target.platform.name if hasattr(current_target, 'test_platform'): info['test_platform'] = current_target.test_platform.name info['roots'] = [{ 'source_root': source_root_package_prefix[0], 'package_prefix': source_root_package_prefix[1] } for source_root_package_prefix in self._source_roots_for_target(current_target)] if classpath_products: info['libraries'] = [self._jar_id(lib) for lib in target_libraries] targets_map[current_target.address.spec] = info for target in targets: process_target(target) jvm_platforms_map = { 'default_platform' : JvmPlatform.global_instance().default_platform.name, 'platforms': { str(platform_name): { 'target_level' : str(platform.target_level), 'source_level' : str(platform.source_level), 'args' : platform.args, } for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items() }, } graph_info = { 'version': self.DEFAULT_EXPORT_VERSION, 'targets': targets_map, 'jvm_platforms': jvm_platforms_map, # `jvm_distributions` are static distribution settings from config, # `preferred_jvm_distributions` are distributions that pants actually uses for the # given platform setting. 'preferred_jvm_distributions': {} } for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items(): preferred_distributions = {} for strict, strict_key in [(True, 'strict'), (False, 'non_strict')]: try: dist = JvmPlatform.preferred_jvm_distribution([platform], strict=strict) preferred_distributions[strict_key] = dist.home except DistributionLocator.Error: pass if preferred_distributions: graph_info['preferred_jvm_distributions'][platform_name] = preferred_distributions if classpath_products: graph_info['libraries'] = self._resolve_jars_info(targets, classpath_products) if python_interpreter_targets_mapping: # NB: We've selected a python interpreter compatible with each python target individually into # the `python_interpreter_targets_mapping`. These python targets may not be compatible, ie: we # could have a python target requiring 'CPython>=2.7<3' (ie: CPython-2.7.x) and another # requiring 'CPython>=3.6'. To pick a default interpreter then from among these two choices # is arbitrary and not to be relied on to work as a default interpreter if ever needed by the # export consumer. # # TODO(John Sirois): consider either eliminating the 'default_interpreter' field and pressing # export consumers to make their own choice of a default (if needed) or else use # `select.select_interpreter_for_targets` and fail fast if there is no interpreter compatible # across all the python targets in-play. # # For now, make our arbitrary historical choice of a default interpreter explicit and use the # lowest version. default_interpreter = min(python_interpreter_targets_mapping.keys()) interpreters_info = {} for interpreter, targets in six.iteritems(python_interpreter_targets_mapping): req_libs = [target for target in Target.closure_for_targets(targets) if has_python_requirements(target)] chroot = self.resolve_requirements(interpreter, req_libs) interpreters_info[str(interpreter.identity)] = { 'binary': interpreter.binary, 'chroot': chroot.path() } graph_info['python_setup'] = { 'default_interpreter': str(default_interpreter.identity), 'interpreters': interpreters_info } return graph_info
def compile( self, ctx, args, dependency_classpath, upstream_analysis, settings, compiler_option_sets, zinc_file_manager, javac_plugin_map, scalac_plugin_map, ): classpath = (ctx.classes_dir.path, ) + tuple( ce.path for ce in dependency_classpath) if self.get_options().capture_classpath: self._record_compile_classpath(classpath, ctx.target, ctx.classes_dir.path) try: distribution = JvmPlatform.preferred_jvm_distribution([settings], strict=True) except DistributionLocator.Error: distribution = JvmPlatform.preferred_jvm_distribution([settings], strict=False) javac_args = [] if settings.args: settings_args = settings.args if any("$JAVA_HOME" in a for a in settings.args): logger.debug( 'Substituting "$JAVA_HOME" with "{}" in jvm-platform args.' .format(distribution.home)) settings_args = (a.replace("$JAVA_HOME", distribution.home) for a in settings.args) javac_args.extend(settings_args) javac_args.extend([ # TODO: support -release "-source", str(settings.source_level), "-target", str(settings.target_level), ]) if self.execution_strategy == self.ExecutionStrategy.hermetic: javac_args.extend([ # We need to strip the source root from our output files. Outputting to a directory, and # capturing that directory, does the job. # Unfortunately, javac errors if the directory you pass to -d doesn't exist, and we don't # have a convenient way of making a directory in the output tree, so let's just use the # working directory as our output dir. # This also has the benefit of not needing to strip leading directories from the returned # snapshot. "-d", ".", ]) else: javac_args.extend(["-d", ctx.classes_dir.path]) javac_args.extend(self._javac_plugin_args(javac_plugin_map)) javac_args.extend(args) compiler_option_sets_args = self.get_merged_args_for_compiler_option_sets( compiler_option_sets) javac_args.extend(compiler_option_sets_args) javac_args.extend(["-classpath", ":".join(classpath)]) javac_args.extend(ctx.sources) # From https://docs.oracle.com/javase/8/docs/technotes/tools/windows/javac.html#BHCJEIBB # Wildcards (*) aren’t allowed in these lists (such as for specifying *.java). # Use of the at sign (@) to recursively interpret files isn’t supported. # The -J options aren’t supported because they’re passed to the launcher, # which doesn’t support argument files. j_args = [j_arg for j_arg in javac_args if j_arg.startswith("-J")] safe_javac_args = list(filter(lambda x: x not in j_args, javac_args)) with argfile.safe_args(safe_javac_args, self.get_options()) as batched_args: javac_cmd = [f"{distribution.real_home}/bin/javac"] javac_cmd.extend(j_args) javac_cmd.extend(batched_args) if self.execution_strategy == self.ExecutionStrategy.hermetic: self._execute_hermetic_compile(javac_cmd, ctx) else: with self.context.new_workunit(name="javac", cmd=" ".join(javac_cmd), labels=[WorkUnitLabel.COMPILER ]) as workunit: self.context.log.debug(f"Executing {' '.join(javac_cmd)}") p = subprocess.Popen( javac_cmd, stdout=workunit.output("stdout"), stderr=workunit.output("stderr"), ) return_code = p.wait() workunit.set_outcome( WorkUnit.FAILURE if return_code else WorkUnit.SUCCESS) if return_code: raise TaskError( f"javac exited with return code {return_code}") classes_directory = Path(ctx.classes_dir.path).relative_to( get_buildroot()) self.context._scheduler.materialize_directory( DirectoryToMaterialize( self.post_compile_extra_resources_digest( ctx, prepend_post_merge_relative_path=False), path_prefix=str(classes_directory), ), ) self._create_context_jar(ctx)
def compile(self, ctx, args, classpath, upstream_analysis, settings, fatal_warnings, zinc_file_manager, javac_plugin_map, scalac_plugin_map): try: distribution = JvmPlatform.preferred_jvm_distribution([settings], strict=True) except DistributionLocator.Error: distribution = JvmPlatform.preferred_jvm_distribution([settings], strict=False) javac_cmd = ['{}/bin/javac'.format(distribution.real_home)] javac_cmd.extend([ '-classpath', ':'.join(classpath), ]) if settings.args: settings_args = settings.args if any('$JAVA_HOME' in a for a in settings.args): logger.debug('Substituting "$JAVA_HOME" with "{}" in jvm-platform args.' .format(distribution.home)) settings_args = (a.replace('$JAVA_HOME', distribution.home) for a in settings.args) javac_cmd.extend(settings_args) javac_cmd.extend([ # TODO: support -release '-source', str(settings.source_level), '-target', str(settings.target_level), ]) if self.execution_strategy == self.HERMETIC: javac_cmd.extend([ # We need to strip the source root from our output files. Outputting to a directory, and # capturing that directory, does the job. # Unfortunately, javac errors if the directory you pass to -d doesn't exist, and we don't # have a convenient way of making a directory in the output tree, so let's just use the # working directory as our output dir. # This also has the benefit of not needing to strip leading directories from the returned # snapshot. '-d', '.', ]) else: javac_cmd.extend([ '-d', ctx.classes_dir, ]) javac_cmd.extend(self._javac_plugin_args(javac_plugin_map)) javac_cmd.extend(args) if fatal_warnings: javac_cmd.extend(self.get_options().fatal_warnings_enabled_args) else: javac_cmd.extend(self.get_options().fatal_warnings_disabled_args) with argfile.safe_args(ctx.sources, self.get_options()) as batched_sources: javac_cmd.extend(batched_sources) if self.execution_strategy == self.HERMETIC: self._execute_hermetic_compile(javac_cmd, ctx) else: with self.context.new_workunit(name='javac', cmd=' '.join(javac_cmd), labels=[WorkUnitLabel.COMPILER]) as workunit: self.context.log.debug('Executing {}'.format(' '.join(javac_cmd))) p = subprocess.Popen(javac_cmd, stdout=workunit.output('stdout'), stderr=workunit.output('stderr')) return_code = p.wait() workunit.set_outcome(WorkUnit.FAILURE if return_code else WorkUnit.SUCCESS) if return_code: raise TaskError('javac exited with return code {rc}'.format(rc=return_code))
def console_output(self, targets): targets_map = {} resource_target_map = {} ivy_info = None if self.get_options().libraries: ivy_jar_products = self.context.products.get_data("ivy_jar_products") or {} # This product is a list for historical reasons (exclusives groups) but in practice should # have either 0 or 1 entries. ivy_info_list = ivy_jar_products.get("default") if ivy_info_list: assert len(ivy_info_list) == 1, ( "The values in ivy_jar_products should always be length 1," " since we no longer have exclusives groups." ) ivy_info = ivy_info_list[0] ivy_jar_memo = {} python_interpreter_targets_mapping = defaultdict(list) def process_target(current_target): """ :type current_target:pants.base.target.Target """ def get_target_type(target): if target.is_test: return Export.SourceRootTypes.TEST else: if ( isinstance(target, Resources) and target in resource_target_map and resource_target_map[target].is_test ): return Export.SourceRootTypes.TEST_RESOURCE elif isinstance(target, Resources): return Export.SourceRootTypes.RESOURCE else: return Export.SourceRootTypes.SOURCE def get_transitive_jars(jar_lib): """ :type jar_lib: pants.backend.jvm.targets.jar_library.JarLibrary :rtype: twitter.common.collections.orderedset.OrderedSet """ if not ivy_info or not self.get_options().libraries: return OrderedSet() transitive_jars = OrderedSet() for jar in jar_lib.jar_dependencies: transitive_jars.update(ivy_info.get_jars_for_ivy_module(jar, memo=ivy_jar_memo)) return transitive_jars info = { "targets": [], "libraries": [], "roots": [], "target_type": get_target_type(current_target), "is_code_gen": current_target.is_codegen, "pants_target_type": self._get_pants_target_alias(type(current_target)), } if not current_target.is_synthetic: info["globs"] = current_target.globs_relative_to_buildroot() if self.get_options().sources: info["sources"] = list(current_target.sources_relative_to_buildroot()) if isinstance(current_target, PythonRequirementLibrary): reqs = current_target.payload.get_field_value("requirements", set()) """:type : set[pants.backend.python.python_requirement.PythonRequirement]""" info["requirements"] = [req.key for req in reqs] if isinstance(current_target, PythonTarget): interpreter_for_target = self.select_interpreter_for_targets([current_target]) if interpreter_for_target is None: raise TaskError("Unable to find suitable interpreter for {}".format(current_target.address)) python_interpreter_targets_mapping[interpreter_for_target].append(current_target) info["python_interpreter"] = str(interpreter_for_target.identity) target_libraries = OrderedSet() if isinstance(current_target, JarLibrary): target_libraries = get_transitive_jars(current_target) for dep in current_target.dependencies: info["targets"].append(dep.address.spec) if isinstance(dep, JarLibrary): for jar in dep.jar_dependencies: target_libraries.add(IvyModuleRef(jar.org, jar.name, jar.rev)) # Add all the jars pulled in by this jar_library target_libraries.update(get_transitive_jars(dep)) if isinstance(dep, Resources): resource_target_map[dep] = current_target if isinstance(current_target, ScalaLibrary): for dep in current_target.java_sources: info["targets"].append(dep.address.spec) process_target(dep) if isinstance(current_target, JvmTarget): info["excludes"] = [self._exclude_id(exclude) for exclude in current_target.excludes] info["platform"] = current_target.platform.name info["roots"] = map( lambda (source_root, package_prefix): {"source_root": source_root, "package_prefix": package_prefix}, self._source_roots_for_target(current_target), ) if self.get_options().libraries: info["libraries"] = [self._jar_id(lib) for lib in target_libraries] targets_map[current_target.address.spec] = info for target in targets: process_target(target) jvm_platforms_map = { "default_platform": JvmPlatform.global_instance().default_platform.name, "platforms": { str(platform_name): { "target_level": str(platform.target_level), "source_level": str(platform.source_level), "args": platform.args, } for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items() }, } graph_info = { "version": self.DEFAULT_EXPORT_VERSION, "targets": targets_map, "jvm_platforms": jvm_platforms_map, } jvm_distributions = DistributionLocator.global_instance().all_jdk_paths() if jvm_distributions: graph_info["jvm_distributions"] = jvm_distributions if self.get_options().libraries: graph_info["libraries"] = self._resolve_jars_info() if python_interpreter_targets_mapping: default_interpreter = self.interpreter_cache.select_interpreter(python_interpreter_targets_mapping.keys())[ 0 ] interpreters_info = {} for interpreter, targets in python_interpreter_targets_mapping.iteritems(): chroot = self.cached_chroot(interpreter=interpreter, pex_info=PexInfo.default(), targets=targets) interpreters_info[str(interpreter.identity)] = {"binary": interpreter.binary, "chroot": chroot.path()} graph_info["python_setup"] = { "default_interpreter": str(default_interpreter.identity), "interpreters": interpreters_info, } if self.format: return json.dumps(graph_info, indent=4, separators=(",", ": ")).splitlines() else: return [json.dumps(graph_info)]
def _run_tests(self, tests_to_targets): if self._coverage: extra_jvm_options = self._coverage.extra_jvm_options classpath_prepend = self._coverage.classpath_prepend classpath_append = self._coverage.classpath_append else: extra_jvm_options = [] classpath_prepend = () classpath_append = () tests_by_properties = self._tests_by_properties( tests_to_targets, self._infer_workdir, lambda target: target.test_platform, lambda target: target.payload.extra_jvm_options, lambda target: target.payload.extra_env_vars, lambda target: target.concurrency, lambda target: target.threads ) # the below will be None if not set, and we'll default back to runtime_classpath classpath_product = self.context.products.get_data('instrument_classpath') result = 0 for properties, tests in tests_by_properties.items(): (workdir, platform, target_jvm_options, target_env_vars, concurrency, threads) = properties for batch in self._partition(tests): # Batches of test classes will likely exist within the same targets: dedupe them. relevant_targets = set(map(tests_to_targets.get, batch)) complete_classpath = OrderedSet() complete_classpath.update(classpath_prepend) complete_classpath.update(self.tool_classpath('junit')) complete_classpath.update(self.classpath(relevant_targets, classpath_product=classpath_product)) complete_classpath.update(classpath_append) distribution = JvmPlatform.preferred_jvm_distribution([platform], self._strict_jvm_version) # Override cmdline args with values from junit_test() target that specify concurrency: args = self._args + [u'-xmlreport'] if concurrency is not None: args = remove_arg(args, '-default-parallel') if concurrency == junit_tests.CONCURRENCY_SERIAL: args = ensure_arg(args, '-default-concurrency', param='SERIAL') elif concurrency == junit_tests.CONCURRENCY_PARALLEL_CLASSES: args = ensure_arg(args, '-default-concurrency', param='PARALLEL_CLASSES') elif concurrency == junit_tests.CONCURRENCY_PARALLEL_METHODS: args = ensure_arg(args, '-default-concurrency', param='PARALLEL_METHODS') elif concurrency == junit_tests.CONCURRENCY_PARALLEL_CLASSES_AND_METHODS: args = ensure_arg(args, '-default-concurrency', param='PARALLEL_CLASSES_AND_METHODS') if threads is not None: args = remove_arg(args, '-parallel-threads', has_param=True) args += ['-parallel-threads', str(threads)] with argfile.safe_args(batch, self.get_options()) as batch_tests: self.context.log.debug('CWD = {}'.format(workdir)) self.context.log.debug('platform = {}'.format(platform)) with environment_as(**dict(target_env_vars)): result += abs(self._spawn_and_wait( executor=SubprocessExecutor(distribution), distribution=distribution, classpath=complete_classpath, main=JUnitRun._MAIN, jvm_options=self.jvm_options + extra_jvm_options + list(target_jvm_options), args=args + batch_tests, workunit_factory=self.context.new_workunit, workunit_name='run', workunit_labels=[WorkUnitLabel.TEST], cwd=workdir, synthetic_jar_dir=self.workdir, create_synthetic_jar=self.synthetic_classpath, )) if result != 0 and self._fail_fast: break if result != 0: failed_targets_and_tests = self._get_failed_targets(tests_to_targets) failed_targets = sorted(failed_targets_and_tests, key=lambda target: target.address.spec) error_message_lines = [] if self._failure_summary: for target in failed_targets: error_message_lines.append('\n{0}{1}'.format(' '*4, target.address.spec)) for test in sorted(failed_targets_and_tests[target]): error_message_lines.append('{0}{1}'.format(' '*8, test)) error_message_lines.append( '\njava {main} ... exited non-zero ({code}); {failed} failed {targets}.' .format(main=JUnitRun._MAIN, code=result, failed=len(failed_targets), targets=pluralize(len(failed_targets), 'target')) ) raise TestFailedTaskError('\n'.join(error_message_lines), failed_targets=list(failed_targets))
def execute(self): if JvmPlatform.global_instance().get_options().compiler == 'javac': return super(JavacCompile, self).execute()
def test_strict_usage(self): init_subsystem( JvmPlatform, options={ "jvm-platform": { "platforms": { "default-platform": {"target": "9"}, "8-platform": {"target": "8"}, "9-platform": {"target": "9"}, "strict-8-platform": {"target": "8", "strict": True}, "strict-9-platform": {"target": "9", "strict": True}, }, "default_platform": "default-platform", "default_runtime_platform": None, } }, ) instance = JvmPlatform.global_instance() strict_8_platform = instance.get_platform_by_name("strict-8-platform") default_9_platform = instance.default_platform # TODO maybe this should use the runtime platform assert instance._preferred_jvm_distribution_args([]) == { "jdk": False, } assert JvmPlatform._preferred_jvm_distribution_args([default_9_platform]) == { "minimum_version": Revision.lenient("9.0.0"), "maximum_version": None, "jdk": False, } assert JvmPlatform._preferred_jvm_distribution_args([default_9_platform], strict=True) == { "minimum_version": Revision.lenient("9.0.0"), "maximum_version": Revision.lenient("9.0.9999"), "jdk": False, } assert instance._preferred_jvm_distribution_args([strict_8_platform]) == { "minimum_version": Revision.lenient("1.8.0"), "maximum_version": Revision.lenient("1.8.9999"), "jdk": False, } assert instance._preferred_jvm_distribution_args([strict_8_platform], strict=False) == { "minimum_version": Revision.lenient("1.8.0"), "maximum_version": None, "jdk": False, } with self.assertRaisesRegex( JvmPlatform.IncompatiblePlatforms, "lenient platform with higher minimum version, 9, than strict requirement of 1.8", ): # requested strict 8 & lenient 9. # fail because 9 is lower bound JvmPlatform._preferred_jvm_distribution_args( [ instance.get_platform_by_name("9-platform"), instance.get_platform_by_name("strict-8-platform"), ] ) with self.assertRaisesRegex( JvmPlatform.IncompatiblePlatforms, "Multiple strict platforms with differing target releases were found: 1.8, 9", ): # two different strict platforms can't work JvmPlatform._preferred_jvm_distribution_args( [ instance.get_platform_by_name("strict-9-platform"), instance.get_platform_by_name("strict-8-platform"), ] ) # two of the same strict platform thumbs up assert JvmPlatform._preferred_jvm_distribution_args( [ instance.get_platform_by_name("strict-8-platform"), instance.get_platform_by_name("strict-8-platform"), ] ) == { "minimum_version": Revision.lenient("1.8.0"), "maximum_version": Revision.lenient("1.8.9999"), "jdk": False, } # strict highest, matching highest non-strict, other non-strict assert JvmPlatform._preferred_jvm_distribution_args( [ instance.get_platform_by_name("strict-9-platform"), instance.get_platform_by_name("9-platform"), instance.get_platform_by_name("8-platform"), ] ) == { "minimum_version": Revision.lenient("9.0.0"), "maximum_version": Revision.lenient("9.0.9999"), "jdk": False, }
def generate_project(self, project): def linked_folder_id(source_set): return source_set.source_base.replace(os.path.sep, '.') def base_path(source_set): return os.path.join(source_set.root_dir, source_set.source_base, source_set.path) def create_source_base_template(source_set): source_base = base_path(source_set) return SourceBase(id=linked_folder_id(source_set), path=source_base) source_sets = project.sources[:] if project.has_python: source_sets.extend(project.py_sources) source_bases = frozenset(map(create_source_base_template, source_sets)) libs = [] def add_jarlibs(classpath_entries): for classpath_entry in classpath_entries: libs.append((classpath_entry.jar, classpath_entry.source_jar)) add_jarlibs(project.internal_jars) add_jarlibs(project.external_jars) scala_full_version = scala_platform.scala_build_info[ self.context.options['scala-platform']['version']].full_version scala = { 'language_level': scala_full_version, 'compiler_classpath': project.scala_compiler_classpath } outdir = os.path.abspath(self.ensime_output_dir) if not os.path.exists(outdir): os.makedirs(outdir) java_platform = JvmPlatform.global_instance().default_platform jdk_home = JvmPlatform.preferred_jvm_distribution([java_platform], strict=True).home configured_project = { 'name': self.project_name, 'java_home': jdk_home, 'scala': scala, 'source_bases': source_bases, 'has_tests': project.has_tests, 'internal_jars': [cp_entry.jar for cp_entry in project.internal_jars], 'internal_source_jars': [ cp_entry.source_jar for cp_entry in project.internal_jars if cp_entry.source_jar ], 'external_jars': [cp_entry.jar for cp_entry in project.external_jars], 'external_javadoc_jars': [ cp_entry.javadoc_jar for cp_entry in project.external_jars if cp_entry.javadoc_jar ], 'external_source_jars': [ cp_entry.source_jar for cp_entry in project.external_jars if cp_entry.source_jar ], 'libs': libs, 'outdir': os.path.relpath(outdir, get_buildroot()), 'root_dir': get_buildroot(), 'cache_dir': os.path.join(self.cwd, '.ensime_cache') } with safe_open(self.project_filename, 'w') as output: output.write( self._renderer.render_name(_TEMPLATE, {'project': configured_project})) print('\nGenerated ensime project at {}{}'.format( self.gen_project_workdir, os.sep))
def generate_targets_map(self, targets, runtime_classpath): """Generates a dictionary containing all pertinent information about the target graph. The return dictionary is suitable for serialization by json.dumps. :param targets: The list of targets to generate the map for. :param classpath_products: Optional classpath_products. If not provided when the --libraries option is `True`, this task will perform its own jar resolution. """ def _get_target_type(tgt): def is_test(t): return isinstance(t, JUnitTests) if is_test(tgt): return SourceRootTypes.TEST else: if (isinstance(tgt, Resources) and tgt in resource_target_map and is_test(resource_target_map[tgt])): return SourceRootTypes.TEST_RESOURCE elif isinstance(tgt, Resources): return SourceRootTypes.RESOURCE else: return SourceRootTypes.SOURCE targets_map = {} resource_target_map = {} target_roots_set = set(self.context.target_roots) def process_target(current_target): """ :type current_target:pants.build_graph.target.Target """ info = { # this means 'dependencies' 'targets': [], 'libraries': [], 'roots': [], 'id': current_target.id, 'target_type': _get_target_type(current_target), 'is_synthetic': current_target.is_synthetic, 'pants_target_type': self._get_pants_target_alias(type(current_target)), 'is_target_root': current_target in target_roots_set, 'transitive': current_target.transitive, 'scope': str(current_target.scope) } if not current_target.is_synthetic: info['globs'] = current_target.globs_relative_to_buildroot() def iter_transitive_jars(jar_lib): """ :type jar_lib: :class:`pants.backend.jvm.targets.jar_library.JarLibrary` :rtype: :class:`collections.Iterator` of :class:`pants.java.jar.M2Coordinate` """ if runtime_classpath: jar_products = runtime_classpath.get_artifact_classpath_entries_for_targets((jar_lib,)) for _, jar_entry in jar_products: coordinate = jar_entry.coordinate # We drop classifier and type_ since those fields are represented in the global # libraries dict and here we just want the key into that dict (see `_jar_id`). yield M2Coordinate(org=coordinate.org, name=coordinate.name, rev=coordinate.rev) target_libraries = OrderedSet() if isinstance(current_target, JarLibrary): target_libraries = OrderedSet(iter_transitive_jars(current_target)) for dep in current_target.dependencies: info['targets'].append(dep.address.spec) if isinstance(dep, JarLibrary): for jar in dep.jar_dependencies: target_libraries.add(M2Coordinate(jar.org, jar.name, jar.rev)) # Add all the jars pulled in by this jar_library target_libraries.update(iter_transitive_jars(dep)) if isinstance(dep, Resources): resource_target_map[dep] = current_target if isinstance(current_target, ScalaLibrary): for dep in current_target.java_sources: info['targets'].append(dep.address.spec) if isinstance(current_target, JvmTarget): info['excludes'] = [self._exclude_id(exclude) for exclude in current_target.excludes] info['platform'] = current_target.platform.name if hasattr(current_target, 'test_platform'): info['test_platform'] = current_target.test_platform.name if runtime_classpath: info['libraries'].extend(self._jar_id(lib) for lib in target_libraries) if current_target in target_roots_set: info['roots'] = [{ 'source_root': os.path.realpath(source_root_package_prefix[0]), 'package_prefix': source_root_package_prefix[1] } for source_root_package_prefix in self._source_roots_for_target(current_target)] targets_map[current_target.address.spec] = info additional_java_targets = [] for t in targets: if isinstance(t, ScalaLibrary): additional_java_targets.extend(t.java_sources) targets.extend(additional_java_targets) for target in targets: process_target(target) scala_platform = ScalaPlatform.global_instance() scala_platform_map = { 'scala_version': scala_platform.version, 'compiler_classpath': [ cp_entry.path for cp_entry in scala_platform.compiler_classpath_entries(self.context.products) ], } jvm_platforms_map = { 'default_platform': JvmPlatform.global_instance().default_platform.name, 'platforms': { str(platform_name): { 'target_level': str(platform.target_level), 'source_level': str(platform.source_level), 'args': platform.args, } for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items()}, } graph_info = { 'version': DEFAULT_EXPORT_VERSION, 'targets': targets_map, 'jvm_platforms': jvm_platforms_map, 'scala_platform': scala_platform_map, # `jvm_distributions` are static distribution settings from config, # `preferred_jvm_distributions` are distributions that pants actually uses for the # given platform setting. 'preferred_jvm_distributions': {} } for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items(): preferred_distributions = {} for strict, strict_key in [(True, 'strict'), (False, 'non_strict')]: try: dist = JvmPlatform.preferred_jvm_distribution([platform], strict=strict) preferred_distributions[strict_key] = dist.home except DistributionLocator.Error: pass if preferred_distributions: graph_info['preferred_jvm_distributions'][platform_name] = preferred_distributions def zip_sources(target, location, suffix='.jar'): with temporary_file(root_dir=location, cleanup=False, suffix=suffix) as f: with zipfile.ZipFile(f, 'a') as zip_file: for src_from_source_root, src_from_build_root in zip(target.sources_relative_to_source_root(), target.sources_relative_to_buildroot()): zip_file.write(os.path.join(get_buildroot(), src_from_build_root), src_from_source_root) return f if runtime_classpath: graph_info['libraries'] = self._resolve_jars_info(targets, runtime_classpath) # Using resolved path in preparation for VCFS. resource_jar_root = os.path.realpath(self.versioned_workdir) for t in targets: target_type = _get_target_type(t) # If it is a target root or it is already a jar_library target, then no-op. if t in target_roots_set or targets_map[t.address.spec]['pants_target_type'] == 'jar_library': continue if target_type == SourceRootTypes.RESOURCE or target_type == SourceRootTypes.TEST_RESOURCE: # yic assumed that the cost to fingerprint the target may not be that lower than # just zipping up the resources anyway. jarred_resources = zip_sources(t, resource_jar_root) targets_map[t.address.spec]['pants_target_type'] = 'jar_library' targets_map[t.address.spec]['libraries'] = [t.id] graph_info['libraries'][t.id]['default'] = jarred_resources.name continue targets_map[t.address.spec]['pants_target_type'] = 'jar_library' targets_map[t.address.spec]['libraries'] = [t.id] jar_products = runtime_classpath.get_for_target(t) for conf, jar_entry in jar_products: # TODO(yic): check --compile-rsc-use-classpath-jars is enabled. # If not, zip up the classes/ dir here. if 'z.jar' in jar_entry: graph_info['libraries'][t.id][conf] = jar_entry if self.get_options().sources: # NB: We create the jar in the same place as we create the resources # (as opposed to where we store the z.jar), because the path to the z.jar depends # on tasks outside of this one. # In addition to that, we may not want to depend on z.jar existing to export source jars. jarred_sources = zip_sources(t, resource_jar_root, suffix='-sources.jar') graph_info['libraries'][t.id]['sources'] = jarred_sources.name return graph_info
def console_output(self, targets): targets_map = {} resource_target_map = {} ivy_info = None if self.get_options().libraries: ivy_jar_products = self.context.products.get_data( 'ivy_jar_products') or {} # This product is a list for historical reasons (exclusives groups) but in practice should # have either 0 or 1 entries. ivy_info_list = ivy_jar_products.get('default') if ivy_info_list: assert len(ivy_info_list) == 1, ( 'The values in ivy_jar_products should always be length 1,' ' since we no longer have exclusives groups.') ivy_info = ivy_info_list[0] ivy_jar_memo = {} python_interpreter_targets_mapping = defaultdict(list) def process_target(current_target): """ :type current_target:pants.base.target.Target """ def get_target_type(target): if target.is_test: return Export.SourceRootTypes.TEST else: if (isinstance(target, Resources) and target in resource_target_map and resource_target_map[target].is_test): return Export.SourceRootTypes.TEST_RESOURCE elif isinstance(target, Resources): return Export.SourceRootTypes.RESOURCE else: return Export.SourceRootTypes.SOURCE def get_transitive_jars(jar_lib): """ :type jar_lib: pants.backend.jvm.targets.jar_library.JarLibrary :rtype: twitter.common.collections.orderedset.OrderedSet """ if not ivy_info or not self.get_options().libraries: return OrderedSet() transitive_jars = OrderedSet() for jar in jar_lib.jar_dependencies: transitive_jars.update( ivy_info.get_jars_for_ivy_module(jar, memo=ivy_jar_memo)) return transitive_jars info = { 'targets': [], 'libraries': [], 'roots': [], 'target_type': get_target_type(current_target), 'is_code_gen': current_target.is_codegen, 'pants_target_type': self._get_pants_target_alias(type(current_target)) } if not current_target.is_synthetic: info['globs'] = current_target.globs_relative_to_buildroot() if self.get_options().sources: info['sources'] = list( current_target.sources_relative_to_buildroot()) if isinstance(current_target, PythonRequirementLibrary): reqs = current_target.payload.get_field_value( 'requirements', set()) """:type : set[pants.backend.python.python_requirement.PythonRequirement]""" info['requirements'] = [req.key for req in reqs] if isinstance(current_target, PythonTarget): interpreter_for_target = self.select_interpreter_for_targets( [current_target]) if interpreter_for_target is None: raise TaskError( 'Unable to find suitable interpreter for {}'.format( current_target.address)) python_interpreter_targets_mapping[ interpreter_for_target].append(current_target) info['python_interpreter'] = str( interpreter_for_target.identity) target_libraries = OrderedSet() if isinstance(current_target, JarLibrary): target_libraries = get_transitive_jars(current_target) for dep in current_target.dependencies: info['targets'].append(dep.address.spec) if isinstance(dep, JarLibrary): for jar in dep.jar_dependencies: target_libraries.add( IvyModuleRef(jar.org, jar.name, jar.rev)) # Add all the jars pulled in by this jar_library target_libraries.update(get_transitive_jars(dep)) if isinstance(dep, Resources): resource_target_map[dep] = current_target if isinstance(current_target, ScalaLibrary): for dep in current_target.java_sources: info['targets'].append(dep.address.spec) process_target(dep) if isinstance(current_target, JvmTarget): info['excludes'] = [ self._exclude_id(exclude) for exclude in current_target.excludes ] info['platform'] = current_target.platform.name info['roots'] = map( lambda (source_root, package_prefix): { 'source_root': source_root, 'package_prefix': package_prefix }, self._source_roots_for_target(current_target)) if self.get_options().libraries: info['libraries'] = [ self._jar_id(lib) for lib in target_libraries ] targets_map[current_target.address.spec] = info for target in targets: process_target(target) jvm_platforms_map = { 'default_platform': JvmPlatform.global_instance().default_platform.name, 'platforms': { str(platform_name): { 'target_level': str(platform.target_level), 'source_level': str(platform.source_level), 'args': platform.args, } for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items() } } graph_info = { 'version': self.DEFAULT_EXPORT_VERSION, 'targets': targets_map, 'jvm_platforms': jvm_platforms_map, } jvm_distributions = DistributionLocator.global_instance( ).all_jdk_paths() if jvm_distributions: graph_info['jvm_distributions'] = jvm_distributions if self.get_options().libraries: graph_info['libraries'] = self._resolve_jars_info() if python_interpreter_targets_mapping: default_interpreter = self.interpreter_cache.select_interpreter( python_interpreter_targets_mapping.keys())[0] interpreters_info = {} for interpreter, targets in python_interpreter_targets_mapping.iteritems( ): chroot = self.cached_chroot(interpreter=interpreter, pex_info=PexInfo.default(), targets=targets) interpreters_info[str(interpreter.identity)] = { 'binary': interpreter.binary, 'chroot': chroot.path() } graph_info['python_setup'] = { 'default_interpreter': str(default_interpreter.identity), 'interpreters': interpreters_info } if self.format: return json.dumps(graph_info, indent=4, separators=(',', ': ')).splitlines() else: return [json.dumps(graph_info)]
def get_preferred_distribution(platform, strict): try: return JvmPlatform.preferred_jvm_distribution([platform], strict=strict) except DistributionLocator.Error: return None
def _run_tests(self, test_registry, output_dir, coverage=None): tests_by_properties = test_registry.index( lambda tgt: tgt.cwd if tgt.cwd is not None else self._working_dir, lambda tgt: tgt.platform) # the below will be None if not set, and we'll default back to runtime_classpath classpath_product = self.context.products.get_data('instrument_classpath') result = 0 base_args = self.get_options().args for properties, tests in tests_by_properties.items(): (cwd, platform) = properties for batch in self._partition(tests, test_registry): # Batches of test classes will likely exist within the same targets: dedupe them. relevant_targets = {test_registry.get_owning_target(t) for t in batch} if len(relevant_targets) > 1: raise "oops, should have only had one target" complete_classpath = OrderedSet() # TODO: Include specs2 on the classpath, in case the target doesn't complete_classpath.update(self.classpath(relevant_targets, classpath_product=classpath_product)) distribution = JvmPlatform.preferred_jvm_distribution([platform], self._strict_jvm_version) target_dir = list(relevant_targets)[0].address.spec_path args = base_args[:] opts = self.get_options() if opts.example: args.extend(["ex", opts.example]) if opts.sequential: args.extend(['sequential', 'true']) if opts.show_times: args.extend(['showtimes', 'true']) file_pattern = opts.file_pattern or Specs2Run.TEST_CLASS_PATTERN if '(' not in file_pattern or ')' not in file_pattern: raise TaskError("Test regex must have a group.") args.extend([ "junitxml", "console", "filesrunner.basepath", target_dir, "filesrunner.pattern", file_pattern, "junit.outdir", self.junit_xml_dir ]) self.context.log.debug('CWD = {}'.format(cwd)) self.context.log.debug('platform = {}'.format(platform)) self.context.log.debug('targets = {}'.format(relevant_targets)) self.context.log.debug('args = {}'.format(" ".join(args))) result += abs(self._spawn_and_wait( executor=SubprocessExecutor(distribution), distribution=distribution, classpath=complete_classpath, main=Specs2Run.SPECS2_MAIN, jvm_options=self.jvm_options, args=args, workunit_factory=self.context.new_workunit, workunit_name='run', workunit_labels=[WorkUnitLabel.TEST], cwd=cwd, synthetic_jar_dir=output_dir, create_synthetic_jar=self.synthetic_classpath, )) if result != 0 and self._fail_fast: break if result != 0: failed_targets_and_tests = self._get_failed_targets(test_registry, output_dir) failed_targets = sorted(failed_targets_and_tests, key=lambda target: target.address.spec) error_message_lines = [] if self._failure_summary: for target in failed_targets: error_message_lines.append('\n{0}{1}'.format(' '*4, target.address.spec)) for test in sorted(failed_targets_and_tests[target]): error_message_lines.append('{0}{1}'.format(' '*8, test)) error_message_lines.append( '\njava {main} ... exited non-zero ({code}); {failed} failed {targets}.' .format(main=Specs2Run.SPECS2_MAIN, code=result, failed=len(failed_targets), targets=pluralize(len(failed_targets), 'target')) ) raise TestFailedTaskError('\n'.join(error_message_lines), failed_targets=list(failed_targets))
def execute(self): requested_compiler = JvmPlatform.global_instance().get_options( ).compiler if requested_compiler != self.compiler_name: return deprecated_conditional( lambda: requested_compiler == self.Compiler.ZINC, removal_version='1.20.0.dev0', entity_description='Requested a deprecated compiler: [{}].'.format( requested_compiler), hint_message='Compiler will be defaulted to [{}].'.format( self.compiler_name)) if requested_compiler == self.Compiler.ZINC and self.compiler_name == self.Compiler.RSC: # Issue a deprecation warning (above) and rewrite zinc to rsc, as zinc is being deprecated. JvmPlatform.global_instance().get_options().compiler = RankedValue( 0, self.compiler_name) elif requested_compiler != self.compiler_name: # If the requested compiler is not the one supported by this task, log and abort self.context.log.debug( 'Requested an unsupported compiler [{}], aborting'.format( requested_compiler)) return # In case we have no relevant targets and return early, create the requested product maps. self.create_empty_extra_products() relevant_targets = list(self.context.targets(predicate=self.select)) if not relevant_targets: return # Clone the compile_classpath to the runtime_classpath. classpath_product = self.create_runtime_classpath() fingerprint_strategy = DependencyContext.global_instance( ).create_fingerprint_strategy(classpath_product) # Note, JVM targets are validated (`vts.update()`) as they succeed. As a result, # we begin writing artifacts out to the cache immediately instead of waiting for # all targets to finish. with self.invalidated(relevant_targets, invalidate_dependents=True, fingerprint_strategy=fingerprint_strategy, topological_order=True) as invalidation_check: compile_contexts = { vt.target: self.create_compile_context(vt.target, vt.results_dir) for vt in invalidation_check.all_vts } self.do_compile( invalidation_check, compile_contexts, classpath_product, ) if not self.get_options().use_classpath_jars: # Once compilation has completed, replace the classpath entry for each target with # its jar'd representation. for ccs in compile_contexts.values(): cc = self.select_runtime_context(ccs) for conf in self._confs: classpath_product.remove_for_target( cc.target, [(conf, cc.classes_dir)]) classpath_product.add_for_target( cc.target, [(conf, cc.jar_file)])
def generate_targets_map(self, targets, classpath_products=None): """Generates a dictionary containing all pertinent information about the target graph. The return dictionary is suitable for serialization by json.dumps. :param targets: The list of targets to generate the map for. :param classpath_products: Optional classpath_products. If not provided when the --libraries option is `True`, this task will perform its own jar resolution. """ targets_map = {} resource_target_map = {} python_interpreter_targets_mapping = defaultdict(list) if self.get_options().libraries: # NB(gmalmquist): This supports mocking the classpath_products in tests. if classpath_products is None: classpath_products = self.resolve_jars(targets) else: classpath_products = None def process_target(current_target): """ :type current_target:pants.build_graph.target.Target """ def get_target_type(target): if target.is_test: return ExportTask.SourceRootTypes.TEST else: if (isinstance(target, Resources) and target in resource_target_map and resource_target_map[target].is_test): return ExportTask.SourceRootTypes.TEST_RESOURCE elif isinstance(target, Resources): return ExportTask.SourceRootTypes.RESOURCE else: return ExportTask.SourceRootTypes.SOURCE info = { 'targets': [], 'libraries': [], 'roots': [], 'id': current_target.id, 'target_type': get_target_type(current_target), # NB: is_code_gen should be removed when export format advances to 1.1.0 or higher 'is_code_gen': current_target.is_codegen, 'is_synthetic': current_target.is_synthetic, 'pants_target_type': self._get_pants_target_alias(type(current_target)) } if not current_target.is_synthetic: info['globs'] = current_target.globs_relative_to_buildroot() if self.get_options().sources: info['sources'] = list(current_target.sources_relative_to_buildroot()) if isinstance(current_target, PythonRequirementLibrary): reqs = current_target.payload.get_field_value('requirements', set()) """:type : set[pants.backend.python.python_requirement.PythonRequirement]""" info['requirements'] = [req.key for req in reqs] if isinstance(current_target, PythonTarget): interpreter_for_target = self.select_interpreter_for_targets([current_target]) if interpreter_for_target is None: raise TaskError('Unable to find suitable interpreter for {}' .format(current_target.address)) python_interpreter_targets_mapping[interpreter_for_target].append(current_target) info['python_interpreter'] = str(interpreter_for_target.identity) def iter_transitive_jars(jar_lib): """ :type jar_lib: :class:`pants.backend.jvm.targets.jar_library.JarLibrary` :rtype: :class:`collections.Iterator` of :class:`pants.backend.jvm.jar_dependency_utils.M2Coordinate` """ if classpath_products: jar_products = classpath_products.get_artifact_classpath_entries_for_targets((jar_lib,)) for _, jar_entry in jar_products: coordinate = jar_entry.coordinate # We drop classifier and type_ since those fields are represented in the global # libraries dict and here we just want the key into that dict (see `_jar_id`). yield M2Coordinate(org=coordinate.org, name=coordinate.name, rev=coordinate.rev) target_libraries = OrderedSet() if isinstance(current_target, JarLibrary): target_libraries = OrderedSet(iter_transitive_jars(current_target)) for dep in current_target.dependencies: info['targets'].append(dep.address.spec) if isinstance(dep, JarLibrary): for jar in dep.jar_dependencies: target_libraries.add(M2Coordinate(jar.org, jar.name, jar.rev)) # Add all the jars pulled in by this jar_library target_libraries.update(iter_transitive_jars(dep)) if isinstance(dep, Resources): resource_target_map[dep] = current_target if isinstance(current_target, ScalaLibrary): for dep in current_target.java_sources: info['targets'].append(dep.address.spec) process_target(dep) if isinstance(current_target, JvmTarget): info['excludes'] = [self._exclude_id(exclude) for exclude in current_target.excludes] info['platform'] = current_target.platform.name info['roots'] = map(lambda (source_root, package_prefix): { 'source_root': source_root, 'package_prefix': package_prefix }, self._source_roots_for_target(current_target)) if classpath_products: info['libraries'] = [self._jar_id(lib) for lib in target_libraries] targets_map[current_target.address.spec] = info for target in targets: process_target(target) jvm_platforms_map = { 'default_platform' : JvmPlatform.global_instance().default_platform.name, 'platforms': { str(platform_name): { 'target_level' : str(platform.target_level), 'source_level' : str(platform.source_level), 'args' : platform.args, } for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items() } } graph_info = { 'version': self.DEFAULT_EXPORT_VERSION, 'targets': targets_map, 'jvm_platforms': jvm_platforms_map, } jvm_distributions = DistributionLocator.global_instance().all_jdk_paths() if jvm_distributions: graph_info['jvm_distributions'] = jvm_distributions if classpath_products: graph_info['libraries'] = self._resolve_jars_info(targets, classpath_products) if python_interpreter_targets_mapping: interpreters = self.interpreter_cache.select_interpreter( python_interpreter_targets_mapping.keys()) default_interpreter = interpreters[0] interpreters_info = {} for interpreter, targets in six.iteritems(python_interpreter_targets_mapping): chroot = self.cached_chroot( interpreter=interpreter, pex_info=PexInfo.default(), targets=targets ) interpreters_info[str(interpreter.identity)] = { 'binary': interpreter.binary, 'chroot': chroot.path() } graph_info['python_setup'] = { 'default_interpreter': str(default_interpreter.identity), 'interpreters': interpreters_info } return graph_info
def generate_targets_map(self, targets, classpath_products=None): """Generates a dictionary containing all pertinent information about the target graph. The return dictionary is suitable for serialization by json.dumps. :param targets: The list of targets to generate the map for. :param classpath_products: Optional classpath_products. If not provided when the --libraries option is `True`, this task will perform its own jar resolution. """ targets_map = {} resource_target_map = {} python_interpreter_targets_mapping = defaultdict(list) if self.get_options().libraries: # NB(gmalmquist): This supports mocking the classpath_products in tests. if classpath_products is None: classpath_products = self.resolve_jars(targets) else: classpath_products = None def process_target(current_target): """ :type current_target:pants.build_graph.target.Target """ def get_target_type(target): if target.is_test: return ExportTask.SourceRootTypes.TEST else: if (isinstance(target, Resources) and target in resource_target_map and resource_target_map[target].is_test): return ExportTask.SourceRootTypes.TEST_RESOURCE elif isinstance(target, Resources): return ExportTask.SourceRootTypes.RESOURCE else: return ExportTask.SourceRootTypes.SOURCE info = { 'targets': [], 'libraries': [], 'roots': [], 'target_type': get_target_type(current_target), 'is_code_gen': current_target.is_codegen, 'pants_target_type': self._get_pants_target_alias(type(current_target)) } if not current_target.is_synthetic: info['globs'] = current_target.globs_relative_to_buildroot() if self.get_options().sources: info['sources'] = list(current_target.sources_relative_to_buildroot()) if isinstance(current_target, PythonRequirementLibrary): reqs = current_target.payload.get_field_value('requirements', set()) """:type : set[pants.backend.python.python_requirement.PythonRequirement]""" info['requirements'] = [req.key for req in reqs] if isinstance(current_target, PythonTarget): interpreter_for_target = self.select_interpreter_for_targets([current_target]) if interpreter_for_target is None: raise TaskError('Unable to find suitable interpreter for {}' .format(current_target.address)) python_interpreter_targets_mapping[interpreter_for_target].append(current_target) info['python_interpreter'] = str(interpreter_for_target.identity) def iter_transitive_jars(jar_lib): """ :type jar_lib: :class:`pants.backend.jvm.targets.jar_library.JarLibrary` :rtype: :class:`collections.Iterator` of :class:`pants.backend.jvm.jar_dependency_utils.M2Coordinate` """ if classpath_products: jar_products = classpath_products.get_artifact_classpath_entries_for_targets((jar_lib,)) for _, jar_entry in jar_products: coordinate = jar_entry.coordinate # We drop classifier and type_ since those fields are represented in the global # libraries dict and here we just want the key into that dict (see `_jar_id`). yield M2Coordinate(org=coordinate.org, name=coordinate.name, rev=coordinate.rev) target_libraries = OrderedSet() if isinstance(current_target, JarLibrary): target_libraries = OrderedSet(iter_transitive_jars(current_target)) for dep in current_target.dependencies: info['targets'].append(dep.address.spec) if isinstance(dep, JarLibrary): for jar in dep.jar_dependencies: target_libraries.add(M2Coordinate(jar.org, jar.name, jar.rev)) # Add all the jars pulled in by this jar_library target_libraries.update(iter_transitive_jars(dep)) if isinstance(dep, Resources): resource_target_map[dep] = current_target if isinstance(current_target, ScalaLibrary): for dep in current_target.java_sources: info['targets'].append(dep.address.spec) process_target(dep) if isinstance(current_target, JvmTarget): info['excludes'] = [self._exclude_id(exclude) for exclude in current_target.excludes] info['platform'] = current_target.platform.name info['roots'] = map(lambda (source_root, package_prefix): { 'source_root': source_root, 'package_prefix': package_prefix }, self._source_roots_for_target(current_target)) if classpath_products: info['libraries'] = [self._jar_id(lib) for lib in target_libraries] targets_map[current_target.address.spec] = info for target in targets: process_target(target) jvm_platforms_map = { 'default_platform' : JvmPlatform.global_instance().default_platform.name, 'platforms': { str(platform_name): { 'target_level' : str(platform.target_level), 'source_level' : str(platform.source_level), 'args' : platform.args, } for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items() } } graph_info = { 'version': self.DEFAULT_EXPORT_VERSION, 'targets': targets_map, 'jvm_platforms': jvm_platforms_map, } jvm_distributions = DistributionLocator.global_instance().all_jdk_paths() if jvm_distributions: graph_info['jvm_distributions'] = jvm_distributions if classpath_products: graph_info['libraries'] = self._resolve_jars_info(targets, classpath_products) if python_interpreter_targets_mapping: interpreters = self.interpreter_cache.select_interpreter( python_interpreter_targets_mapping.keys()) default_interpreter = interpreters[0] interpreters_info = {} for interpreter, targets in six.iteritems(python_interpreter_targets_mapping): chroot = self.cached_chroot( interpreter=interpreter, pex_info=PexInfo.default(), targets=targets ) interpreters_info[str(interpreter.identity)] = { 'binary': interpreter.binary, 'chroot': chroot.path() } graph_info['python_setup'] = { 'default_interpreter': str(default_interpreter.identity), 'interpreters': interpreters_info } return graph_info
def generate_project(self, project): def linked_folder_id(source_set): return source_set.source_base.replace(os.path.sep, '.') def base_path(source_set): return os.path.join(source_set.root_dir, source_set.source_base, source_set.path) def create_source_base_template(source_set): source_base = base_path(source_set) return SourceBase(id=linked_folder_id(source_set), path=source_base) source_sets = project.sources[:] if project.has_python: source_sets.extend(project.py_sources) source_bases = frozenset(map(create_source_base_template, source_sets)) libs = [] def add_jarlibs(classpath_entries): for classpath_entry in classpath_entries: libs.append((classpath_entry.jar, classpath_entry.source_jar)) add_jarlibs(project.internal_jars) add_jarlibs(project.external_jars) scala_full_version = scala_platform.scala_build_info[ self.context.options['scala-platform']['version']].full_version scala = TemplateData( language_level=scala_full_version, compiler_classpath=project.scala_compiler_classpath) outdir = os.path.abspath(self.ensime_output_dir) if not os.path.exists(outdir): os.makedirs(outdir) java_platform = JvmPlatform.global_instance().default_platform jdk_home = JvmPlatform.preferred_jvm_distribution([java_platform], strict=True).home configured_project = TemplateData( name=self.project_name, java_home=jdk_home, scala=scala, source_bases=source_bases, has_tests=project.has_tests, internal_jars=[cp_entry.jar for cp_entry in project.internal_jars], internal_source_jars=[ cp_entry.source_jar for cp_entry in project.internal_jars if cp_entry.source_jar ], external_jars=[cp_entry.jar for cp_entry in project.external_jars], external_javadoc_jars=[ cp_entry.javadoc_jar for cp_entry in project.external_jars if cp_entry.javadoc_jar ], external_source_jars=[ cp_entry.source_jar for cp_entry in project.external_jars if cp_entry.source_jar ], libs=libs, outdir=os.path.relpath(outdir, get_buildroot()), root_dir=get_buildroot(), cache_dir=os.path.join(self.cwd, '.ensime_cache')) def apply_template(output_path, template_relpath, **template_data): with safe_open(output_path, 'w') as output: Generator(pkgutil.get_data(__name__, template_relpath), **template_data).write(output) apply_template(self.project_filename, self.project_template, project=configured_project) print('\nGenerated ensime project at {}{}'.format( self.gen_project_workdir, os.sep))
def compile(self, ctx, args, dependency_classpath, upstream_analysis, settings, compiler_option_sets, zinc_file_manager, javac_plugin_map, scalac_plugin_map): classpath = (ctx.classes_dir.path,) + tuple(ce.path for ce in dependency_classpath) if self.get_options().capture_classpath: self._record_compile_classpath(classpath, ctx.target, ctx.classes_dir.path) try: distribution = JvmPlatform.preferred_jvm_distribution([settings], strict=True) except DistributionLocator.Error: distribution = JvmPlatform.preferred_jvm_distribution([settings], strict=False) javac_args = [] if settings.args: settings_args = settings.args if any('$JAVA_HOME' in a for a in settings.args): logger.debug('Substituting "$JAVA_HOME" with "{}" in jvm-platform args.' .format(distribution.home)) settings_args = (a.replace('$JAVA_HOME', distribution.home) for a in settings.args) javac_args.extend(settings_args) javac_args.extend([ # TODO: support -release '-source', str(settings.source_level), '-target', str(settings.target_level), ]) if self.execution_strategy == self.HERMETIC: javac_args.extend([ # We need to strip the source root from our output files. Outputting to a directory, and # capturing that directory, does the job. # Unfortunately, javac errors if the directory you pass to -d doesn't exist, and we don't # have a convenient way of making a directory in the output tree, so let's just use the # working directory as our output dir. # This also has the benefit of not needing to strip leading directories from the returned # snapshot. '-d', '.', ]) else: javac_args.extend([ '-d', ctx.classes_dir.path, ]) javac_args.extend(self._javac_plugin_args(javac_plugin_map)) javac_args.extend(args) compiler_option_sets_args = self.get_merged_args_for_compiler_option_sets(compiler_option_sets) javac_args.extend(compiler_option_sets_args) javac_args.extend([ '-classpath', ':'.join(classpath), ]) javac_args.extend(ctx.sources) # From https://docs.oracle.com/javase/8/docs/technotes/tools/windows/javac.html#BHCJEIBB # Wildcards (*) aren’t allowed in these lists (such as for specifying *.java). # Use of the at sign (@) to recursively interpret files isn’t supported. # The -J options aren’t supported because they’re passed to the launcher, # which doesn’t support argument files. j_args = [j_arg for j_arg in javac_args if j_arg.startswith('-J')] safe_javac_args = list(filter(lambda x: x not in j_args, javac_args)) with argfile.safe_args(safe_javac_args, self.get_options()) as batched_args: javac_cmd = ['{}/bin/javac'.format(distribution.real_home)] javac_cmd.extend(j_args) javac_cmd.extend(batched_args) if self.execution_strategy == self.HERMETIC: self._execute_hermetic_compile(javac_cmd, ctx) else: with self.context.new_workunit(name='javac', cmd=' '.join(javac_cmd), labels=[WorkUnitLabel.COMPILER]) as workunit: self.context.log.debug('Executing {}'.format(' '.join(javac_cmd))) p = subprocess.Popen(javac_cmd, stdout=workunit.output('stdout'), stderr=workunit.output('stderr')) return_code = p.wait() workunit.set_outcome(WorkUnit.FAILURE if return_code else WorkUnit.SUCCESS) if return_code: raise TaskError('javac exited with return code {rc}'.format(rc=return_code))
def preferred_jvm_distribution_for_targets(self, targets): return JvmPlatform.preferred_jvm_distribution([ target.platform for target in targets if isinstance(target, JvmTarget) ], self._strict_jvm_version)
def preferred_jvm_distribution(self, platforms, jdk=False, strict=None): return JvmPlatform.preferred_jvm_distribution(platforms, jdk=jdk, strict=strict)
def preferred_jvm_distribution_for_targets(self, targets): return JvmPlatform.preferred_jvm_distribution([target.platform for target in targets if isinstance(target, JvmTarget)], self._strict_jvm_version)
def _run_tests(self, tests_to_targets): if self._coverage: extra_jvm_options = self._coverage.extra_jvm_options classpath_prepend = self._coverage.classpath_prepend classpath_append = self._coverage.classpath_append else: extra_jvm_options = [] classpath_prepend = () classpath_append = () tests_by_properties = self._tests_by_properties( tests_to_targets, self._infer_workdir, lambda target: target.test_platform, lambda target: target.payload.extra_jvm_options, lambda target: target.payload.extra_env_vars, lambda target: target.concurrency, lambda target: target.threads ) # the below will be None if not set, and we'll default back to runtime_classpath classpath_product = self.context.products.get_data('instrument_classpath') result = 0 for properties, tests in tests_by_properties.items(): (workdir, platform, target_jvm_options, target_env_vars, concurrency, threads) = properties for batch in self._partition(tests): # Batches of test classes will likely exist within the same targets: dedupe them. relevant_targets = set(map(tests_to_targets.get, batch)) complete_classpath = OrderedSet() complete_classpath.update(classpath_prepend) complete_classpath.update(self.tool_classpath('junit')) complete_classpath.update(self.classpath(relevant_targets, classpath_product=classpath_product)) complete_classpath.update(classpath_append) distribution = JvmPlatform.preferred_jvm_distribution([platform], self._strict_jvm_version) # Override cmdline args with values from junit_test() target that specify concurrency: args = self._args + [u'-xmlreport'] # TODO(zundel): Combine these together into a single -concurrency choices style argument if concurrency == junit_tests.CONCURRENCY_SERIAL: args = remove_arg(args, '-default-parallel') elif concurrency == junit_tests.CONCURRENCY_PARALLEL_CLASSES: args = ensure_arg(args, '-default-parallel') elif concurrency == junit_tests.CONCURRENCY_PARALLEL_METHODS: self.context.log.warn('Not implemented: parallel_methods') elif concurrency == junit_tests.CONCURRENCY_PARALLEL_BOTH: self.context.log.warn('specifying {} is experimental.'.format(concurrency)) args = ensure_arg(args, '-default-parallel') args = ensure_arg(args, '-parallel-methods') if threads is not None: args = remove_arg(args, '-parallel-threads', has_param=True) args += ['-parallel-threads', str(threads)] with binary_util.safe_args(batch, self.get_options()) as batch_tests: self.context.log.debug('CWD = {}'.format(workdir)) self.context.log.debug('platform = {}'.format(platform)) with environment_as(**dict(target_env_vars)): result += abs(self._spawn_and_wait( executor=SubprocessExecutor(distribution), distribution=distribution, classpath=complete_classpath, main=JUnitRun._MAIN, jvm_options=self.jvm_options + extra_jvm_options + list(target_jvm_options), args=args + batch_tests, workunit_factory=self.context.new_workunit, workunit_name='run', workunit_labels=[WorkUnitLabel.TEST], cwd=workdir, synthetic_jar_dir=self.workdir, create_synthetic_jar=self.synthetic_classpath, )) if result != 0 and self._fail_fast: break if result != 0: failed_targets_and_tests = self._get_failed_targets(tests_to_targets) failed_targets = sorted(failed_targets_and_tests, key=lambda target: target.address.spec) error_message_lines = [] if self._failure_summary: for target in failed_targets: error_message_lines.append('\n{0}{1}'.format(' '*4, target.address.spec)) for test in sorted(failed_targets_and_tests[target]): error_message_lines.append('{0}{1}'.format(' '*8, test)) error_message_lines.append( '\njava {main} ... exited non-zero ({code}); {failed} failed {targets}.' .format(main=JUnitRun._MAIN, code=result, failed=len(failed_targets), targets=pluralize(len(failed_targets), 'target')) ) raise TestFailedTaskError('\n'.join(error_message_lines), failed_targets=list(failed_targets))
def test_synthetic_target_runtime_platform_lookup(self): init_subsystem( JvmPlatform, options={ "jvm-platform": { "platforms": { "default-platform": {"target": "8"}, "default-runtime-platform": {"target": "8"}, "target-platform": {"target": "8"}, "target-runtime-platform": {"target": "8"}, "parent-target-platform": {"target": "8"}, "parent-target-runtime-platform": {"target": "8"}, }, "default_platform": "default-platform", "default_runtime_platform": None, } }, ) just_platform = self.make_target( "//:parent-with-runtime-platform", HasRuntimePlatform, platform="parent-target-platform" ) just_runtime_platform = self.make_target( "//:parent-with-platform", HasRuntimePlatform, runtime_platform="parent-target-runtime-platform", ) synth_none = self.make_target( "//:without-platforms", HasRuntimePlatform, synthetic=True, derived_from=just_runtime_platform, ) synth_just_platform = self.make_target( "//:with-platform", HasRuntimePlatform, synthetic=True, derived_from=just_runtime_platform, platform="target-platform", ) synth_just_runtime = self.make_target( "//:with-runtime-platform", HasRuntimePlatform, synthetic=True, derived_from=just_runtime_platform, runtime_platform="target-runtime-platform", ) synth_both = self.make_target( "//:with-platform-and-runtime-platform", HasRuntimePlatform, synthetic=True, derived_from=just_runtime_platform, platform="target-platform", runtime_platform="target-runtime-platform", ) synth_just_platform_with_parent_same = self.make_target( "//:with-platform-and-platform-parent", HasRuntimePlatform, synthetic=True, derived_from=just_platform, platform="target-platform", ) instance = JvmPlatform.global_instance() assert ( instance.get_runtime_platform_for_target(synth_none).name == "parent-target-runtime-platform" ) assert ( instance.get_runtime_platform_for_target(synth_just_platform).name == "parent-target-runtime-platform" ) assert ( instance.get_runtime_platform_for_target(synth_just_runtime).name == "target-runtime-platform" ) assert ( instance.get_runtime_platform_for_target(synth_both).name == "target-runtime-platform" ) assert ( instance.get_runtime_platform_for_target(synth_just_platform_with_parent_same).name == "default-platform" )
def _run_tests(self, test_registry, output_dir, coverage=None): if coverage: extra_jvm_options = coverage.extra_jvm_options classpath_prepend = coverage.classpath_prepend classpath_append = coverage.classpath_append else: extra_jvm_options = [] classpath_prepend = () classpath_append = () tests_by_properties = test_registry.index( lambda tgt: tgt.cwd if tgt.cwd is not None else self._working_dir, lambda tgt: tgt.test_platform, lambda tgt: tgt.payload.extra_jvm_options, lambda tgt: tgt.payload.extra_env_vars, lambda tgt: tgt.concurrency, lambda tgt: tgt.threads) # the below will be None if not set, and we'll default back to runtime_classpath classpath_product = self.context.products.get_data( 'instrument_classpath') result = 0 for properties, tests in tests_by_properties.items(): (workdir, platform, target_jvm_options, target_env_vars, concurrency, threads) = properties for batch in self._partition(tests): # Batches of test classes will likely exist within the same targets: dedupe them. relevant_targets = { test_registry.get_owning_target(t) for t in batch } complete_classpath = OrderedSet() complete_classpath.update(classpath_prepend) complete_classpath.update( JUnit.global_instance().runner_classpath(self.context)) complete_classpath.update( self.classpath(relevant_targets, classpath_product=classpath_product)) complete_classpath.update(classpath_append) distribution = JvmPlatform.preferred_jvm_distribution( [platform], self._strict_jvm_version) # Override cmdline args with values from junit_test() target that specify concurrency: args = self._args(output_dir) + [u'-xmlreport'] if concurrency is not None: args = remove_arg(args, '-default-parallel') if concurrency == JUnitTests.CONCURRENCY_SERIAL: args = ensure_arg(args, '-default-concurrency', param='SERIAL') elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES: args = ensure_arg(args, '-default-concurrency', param='PARALLEL_CLASSES') elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_METHODS: args = ensure_arg(args, '-default-concurrency', param='PARALLEL_METHODS') elif concurrency == JUnitTests.CONCURRENCY_PARALLEL_CLASSES_AND_METHODS: args = ensure_arg(args, '-default-concurrency', param='PARALLEL_CLASSES_AND_METHODS') if threads is not None: args = remove_arg(args, '-parallel-threads', has_param=True) args += ['-parallel-threads', str(threads)] batch_test_specs = [test.render_test_spec() for test in batch] with argfile.safe_args(batch_test_specs, self.get_options()) as batch_tests: self.context.log.debug('CWD = {}'.format(workdir)) self.context.log.debug('platform = {}'.format(platform)) with environment_as(**dict(target_env_vars)): result += abs( self._spawn_and_wait( executor=SubprocessExecutor(distribution), distribution=distribution, classpath=complete_classpath, main=JUnit.RUNNER_MAIN, jvm_options=self.jvm_options + extra_jvm_options + list(target_jvm_options), args=args + batch_tests, workunit_factory=self.context.new_workunit, workunit_name='run', workunit_labels=[WorkUnitLabel.TEST], cwd=workdir, synthetic_jar_dir=output_dir, create_synthetic_jar=self.synthetic_classpath, )) if result != 0 and self._fail_fast: break if result != 0: def error_handler(parse_error): # Just log and move on since the result is only used to characterize failures, and raising # an error here would just distract from the underlying test failures. self.context.log.error( 'Error parsing test result file {path}: {cause}'.format( path=parse_error.junit_xml_path, cause=parse_error.cause)) target_to_failed_test = parse_failed_targets( test_registry, output_dir, error_handler) failed_targets = sorted(target_to_failed_test, key=lambda t: t.address.spec) error_message_lines = [] if self._failure_summary: for target in failed_targets: error_message_lines.append('\n{indent}{address}'.format( indent=' ' * 4, address=target.address.spec)) for test in sorted(target_to_failed_test[target]): error_message_lines.append( '{indent}{classname}#{methodname}'.format( indent=' ' * 8, classname=test.classname, methodname=test.methodname)) error_message_lines.append( '\njava {main} ... exited non-zero ({code}); {failed} failed {targets}.' .format(main=JUnit.RUNNER_MAIN, code=result, failed=len(failed_targets), targets=pluralize(len(failed_targets), 'target'))) raise ErrorWhileTesting('\n'.join(error_message_lines), failed_targets=list(failed_targets))
def work_for_vts_rsc(vts, ctx): # Double check the cache before beginning compilation hit_cache = self.check_cache(vts, counter) target = ctx.target if not hit_cache: cp_entries = [] # Include the current machine's jdk lib jars. This'll blow up remotely. # We need a solution for that. # Probably something to do with https://github.com/pantsbuild/pants/pull/6346 distribution = JvmPlatform.preferred_jvm_distribution([ctx.target.platform], strict=True) jvm_lib_jars_abs = distribution.find_libs(['rt.jar', 'dt.jar', 'jce.jar', 'tools.jar']) cp_entries.extend(jvm_lib_jars_abs) classpath_abs = self._zinc.compile_classpath( 'rsc_classpath', ctx.target, extra_cp_entries=self._extra_compile_time_classpath) jar_deps = [t for t in DependencyContext.global_instance().dependencies_respecting_strict_deps(target) if isinstance(t, JarLibrary)] metacp_jar_classpath_abs = [y[1] for y in self._metacp_jars_classpath_product.get_for_targets( jar_deps )] jar_jar_paths = {y[1] for y in self.context.products.get_data('rsc_classpath').get_for_targets(jar_deps)} classpath_abs = [c for c in classpath_abs if c not in jar_jar_paths] classpath_rel = fast_relpath_collection(classpath_abs) metacp_jar_classpath_rel = fast_relpath_collection(metacp_jar_classpath_abs) cp_entries.extend(classpath_rel) ctx.ensure_output_dirs_exist() counter_val = str(counter()).rjust(counter.format_length(), b' ') counter_str = '[{}/{}] '.format(counter_val, counter.size) self.context.log.info( counter_str, 'Rsc-ing ', items_to_report_element(ctx.sources, '{} source'.format(self.name())), ' in ', items_to_report_element([t.address.reference() for t in vts.targets], 'target'), ' (', ctx.target.address.spec, ').') tgt, = vts.targets with Timer() as timer: # Step 1: Convert classpath to SemanticDB # --------------------------------------- scalac_classpath_path_entries_abs = self.tool_classpath('workaround-metacp-dependency-classpath') scalac_classpath_path_entries = fast_relpath_collection(scalac_classpath_path_entries_abs) rsc_index_dir = fast_relpath(ctx.rsc_index_dir, get_buildroot()) args = [ '--verbose', # NB: Without this setting, rsc will be missing some symbols # from the scala library. '--include-scala-library-synthetics', # TODO generate these once and cache them # NB: We need to add these extra dependencies in order to be able # to find symbols used by the scalac jars. '--dependency-classpath', os.pathsep.join(scalac_classpath_path_entries + list(jar_jar_paths)), # NB: The directory to dump the semanticdb jars generated by metacp. '--out', rsc_index_dir, os.pathsep.join(cp_entries), ] metacp_wu = self._runtool( 'scala.meta.cli.Metacp', 'metacp', args, distribution, tgt=tgt, input_files=(scalac_classpath_path_entries + classpath_rel), output_dir=rsc_index_dir) metacp_stdout = stdout_contents(metacp_wu) metacp_result = json.loads(metacp_stdout) metai_classpath = self._collect_metai_classpath( metacp_result, classpath_rel, jvm_lib_jars_abs) # Step 1.5: metai Index the semanticdbs # ------------------------------------- self._run_metai_tool(distribution, metai_classpath, rsc_index_dir, tgt) # Step 2: Outline Scala sources into SemanticDB # --------------------------------------------- rsc_outline_dir = fast_relpath(ctx.rsc_outline_dir, get_buildroot()) rsc_out = os.path.join(rsc_outline_dir, 'META-INF/semanticdb/out.semanticdb') safe_mkdir(os.path.join(rsc_outline_dir, 'META-INF/semanticdb')) target_sources = ctx.sources args = [ '-cp', os.pathsep.join(metai_classpath + metacp_jar_classpath_rel), '-out', rsc_out, ] + target_sources self._runtool( 'rsc.cli.Main', 'rsc', args, distribution, tgt=tgt, # TODO pass the input files from the target snapshot instead of the below # input_snapshot = ctx.target.sources_snapshot(scheduler=self.context._scheduler) input_files=target_sources + metai_classpath + metacp_jar_classpath_rel, output_dir=rsc_outline_dir) rsc_classpath = [rsc_outline_dir] # Step 2.5: Postprocess the rsc outputs # TODO: This is only necessary as a workaround for https://github.com/twitter/rsc/issues/199. # Ideally, Rsc would do this on its own. self._run_metai_tool(distribution, rsc_classpath, rsc_outline_dir, tgt, extra_input_files=(rsc_out,)) # Step 3: Convert SemanticDB into an mjar # --------------------------------------- rsc_mjar_file = fast_relpath(ctx.rsc_mjar_file, get_buildroot()) args = [ '-out', rsc_mjar_file, os.pathsep.join(rsc_classpath), ] self._runtool( 'scala.meta.cli.Mjar', 'mjar', args, distribution, tgt=tgt, input_files=( rsc_out, ), output_dir=os.path.dirname(rsc_mjar_file) ) self.context.products.get_data('rsc_classpath').add_for_target( ctx.target, [(conf, ctx.rsc_mjar_file) for conf in self._confs], ) self._record_target_stats(tgt, len(cp_entries), len(target_sources), timer.elapsed, False, 'rsc' ) # Write any additional resources for this target to the target workdir. self.write_extra_resources(ctx) # Update the products with the latest classes. self.register_extra_products_from_contexts([ctx.target], compile_contexts)
def test_platform(self): if self.payload.test_platform: return JvmPlatform.global_instance().get_platform_by_name(self.payload.test_platform) return self.platform
def compile(self, ctx, args, dependency_classpath, upstream_analysis, settings, compiler_option_sets, zinc_file_manager, javac_plugin_map, scalac_plugin_map): classpath = (ctx.classes_dir.path, ) + tuple( ce.path for ce in dependency_classpath) if self.get_options().capture_classpath: self._record_compile_classpath(classpath, ctx.target, ctx.classes_dir.path) try: distribution = JvmPlatform.preferred_jvm_distribution([settings], strict=True) except DistributionLocator.Error: distribution = JvmPlatform.preferred_jvm_distribution([settings], strict=False) javac_cmd = ['{}/bin/javac'.format(distribution.real_home)] javac_cmd.extend([ '-classpath', ':'.join(classpath), ]) if settings.args: settings_args = settings.args if any('$JAVA_HOME' in a for a in settings.args): logger.debug( 'Substituting "$JAVA_HOME" with "{}" in jvm-platform args.' .format(distribution.home)) settings_args = (a.replace('$JAVA_HOME', distribution.home) for a in settings.args) javac_cmd.extend(settings_args) javac_cmd.extend([ # TODO: support -release '-source', str(settings.source_level), '-target', str(settings.target_level), ]) if self.execution_strategy == self.HERMETIC: javac_cmd.extend([ # We need to strip the source root from our output files. Outputting to a directory, and # capturing that directory, does the job. # Unfortunately, javac errors if the directory you pass to -d doesn't exist, and we don't # have a convenient way of making a directory in the output tree, so let's just use the # working directory as our output dir. # This also has the benefit of not needing to strip leading directories from the returned # snapshot. '-d', '.', ]) else: javac_cmd.extend([ '-d', ctx.classes_dir.path, ]) javac_cmd.extend(self._javac_plugin_args(javac_plugin_map)) javac_cmd.extend(args) compiler_option_sets_args = self.get_merged_args_for_compiler_option_sets( compiler_option_sets) javac_cmd.extend(compiler_option_sets_args) with argfile.safe_args(ctx.sources, self.get_options()) as batched_sources: javac_cmd.extend(batched_sources) if self.execution_strategy == self.HERMETIC: self._execute_hermetic_compile(javac_cmd, ctx) else: with self.context.new_workunit(name='javac', cmd=' '.join(javac_cmd), labels=[WorkUnitLabel.COMPILER ]) as workunit: self.context.log.debug('Executing {}'.format( ' '.join(javac_cmd))) p = subprocess.Popen(javac_cmd, stdout=workunit.output('stdout'), stderr=workunit.output('stderr')) return_code = p.wait() workunit.set_outcome( WorkUnit.FAILURE if return_code else WorkUnit.SUCCESS) if return_code: raise TaskError( 'javac exited with return code {rc}'.format( rc=return_code))
def execute(self): if JvmPlatform.global_instance().get_options().compiler == 'zinc': return super(ZincCompile, self).execute()
def _run_tests(self, tests_to_targets): if self._coverage: extra_jvm_options = self._coverage.extra_jvm_options classpath_prepend = self._coverage.classpath_prepend classpath_append = self._coverage.classpath_append else: extra_jvm_options = [] classpath_prepend = () classpath_append = () tests_by_properties = self._tests_by_properties( tests_to_targets, self._infer_workdir, lambda target: target.test_platform, lambda target: target.payload.extra_jvm_options, lambda target: target.payload.extra_env_vars, ) # the below will be None if not set, and we'll default back to runtime_classpath classpath_product = self.context.products.get_data("instrument_classpath") result = 0 for (workdir, platform, target_jvm_options, target_env_vars), tests in tests_by_properties.items(): for batch in self._partition(tests): # Batches of test classes will likely exist within the same targets: dedupe them. relevant_targets = set(map(tests_to_targets.get, batch)) complete_classpath = OrderedSet() complete_classpath.update(classpath_prepend) complete_classpath.update(self.tool_classpath("junit")) complete_classpath.update(self.classpath(relevant_targets, classpath_product=classpath_product)) complete_classpath.update(classpath_append) distribution = JvmPlatform.preferred_jvm_distribution([platform], self._strict_jvm_version) with binary_util.safe_args(batch, self.get_options()) as batch_tests: self.context.log.debug("CWD = {}".format(workdir)) self.context.log.debug("platform = {}".format(platform)) with environment_as(**dict(target_env_vars)): result += abs( self._spawn_and_wait( executor=SubprocessExecutor(distribution), distribution=distribution, classpath=complete_classpath, main=JUnitRun._MAIN, jvm_options=self.jvm_options + extra_jvm_options + list(target_jvm_options), args=self._args + batch_tests + ["-xmlreport"], workunit_factory=self.context.new_workunit, workunit_name="run", workunit_labels=[WorkUnitLabel.TEST], cwd=workdir, synthetic_jar_dir=self.workdir, create_synthetic_jar=self.synthetic_classpath, ) ) if result != 0 and self._fail_fast: break if result != 0: failed_targets_and_tests = self._get_failed_targets(tests_to_targets) failed_targets = sorted(failed_targets_and_tests, key=lambda target: target.address.spec) error_message_lines = [] if self._failure_summary: for target in failed_targets: error_message_lines.append("\n{0}{1}".format(" " * 4, target.address.spec)) for test in sorted(failed_targets_and_tests[target]): error_message_lines.append("{0}{1}".format(" " * 8, test)) error_message_lines.append( "\njava {main} ... exited non-zero ({code}); {failed} failed {targets}.".format( main=JUnitRun._MAIN, code=result, failed=len(failed_targets), targets=pluralize(len(failed_targets), "target"), ) ) raise TestFailedTaskError("\n".join(error_message_lines), failed_targets=list(failed_targets))