def __init__(self, *args, **kwargs): super(JvmRun, self).__init__(*args, **kwargs) self.jvm_args = self.context.config.getlist('jvm-run', 'jvm_args', default=[]) if self.context.options.run_jvmargs: for arg in self.context.options.run_jvmargs: self.jvm_args.extend(safe_shlex_split(arg)) self.args = [] if self.context.options.run_args: for arg in self.context.options.run_args: self.args.extend(safe_shlex_split(arg)) if self.context.options.run_debug: self.jvm_args.extend(JvmDebugConfig.debug_args(self.context.config)) self.confs = self.context.config.getlist('jvm-run', 'confs', default=['default']) self.only_write_cmd_line = self.context.options.only_write_cmd_line
def __init__(self, *args, **kwargs): super(JvmTask, self).__init__(*args, **kwargs) self.jvm_options = [] for jvm_option in self.get_options().jvm_options: self.jvm_options.extend(safe_shlex_split(jvm_option)) if self.get_options().debug: self.jvm_options.extend(JvmDebugConfig.debug_args(self.context.config)) self.args = [] for arg in self.get_options().args: self.args.extend(safe_shlex_split(arg)) self.confs = self.get_options().confs
def execute(self): def is_python_test(target): # Note that we ignore PythonTestSuite, because we'll see the PythonTests targets # it depends on anyway,so if we don't we'll end up running the tests twice. # TODO(benjy): Once we're off the 'build' command we can get rid of python_test_suite, # or make it an alias of dependencies(). return isinstance(target, PythonTests) test_targets = list(filter(is_python_test, self.context.targets())) if test_targets: self.context.release_lock() debug = self.get_options().level == 'debug' args = [] if self.get_options().no_colors else ['--color', 'yes'] for options in self.get_options().options + self.get_passthru_args(): args.extend(safe_shlex_split(options)) test_builder = PythonTestBuilder(context=self.context, targets=test_targets, args=args, interpreter=self.interpreter, fast=self.get_options().fast, debug=debug) with self.context.new_workunit(name='run', labels=[WorkUnit.TOOL, WorkUnit.TEST]) as workunit: # pytest uses py.io.terminalwriter for output. That class detects the terminal # width and attempts to use all of it. However we capture and indent the console # output, leading to weird-looking line wraps. So we trick the detection code # into thinking the terminal window is narrower than it is. cols = os.environ.get('COLUMNS', 80) with environment_as(COLUMNS=str(int(cols) - 30)): stdout = workunit.output('stdout') if workunit else None stderr = workunit.output('stderr') if workunit else None if test_builder.run(stdout=stdout, stderr=stderr): raise TaskError()
def errorprone(self, target): runtime_classpaths = self.context.products.get_data('runtime_classpath') runtime_classpath = [jar for conf, jar in runtime_classpaths.get_for_targets(target.closure(bfs=True))] output_dir = os.path.join(self.workdir, target.id) safe_mkdir(output_dir) runtime_classpath.append(output_dir) args = [ '-classpath', ':'.join(runtime_classpath), '-d', output_dir, ] for opt in self.get_options().command_line_options: args.extend(safe_shlex_split(opt)) args.extend(self.calculate_sources(target)) result = self.runjava(classpath=self.tool_classpath('errorprone'), main=self._ERRORPRONE_MAIN, jvm_options=self.get_options().jvm_options, args=args, workunit_name='errorprone', workunit_labels=[WorkUnitLabel.LINT]) self.context.log.debug('java {main} ... exited with result ({result})'.format( main=self._ERRORPRONE_MAIN, result=result)) return result
def __init__(self, *args, **kwargs): super(JavaCompile, self).__init__(*args, **kwargs) self.set_distribution(jdk=True) self._buildroot = get_buildroot() self._depfile = os.path.join(self._analysis_dir, 'global_depfile') self._jmake_bootstrap_key = 'jmake' external_tools = self.context.config.getlist('java-compile', 'jmake-bootstrap-tools', default=['//:jmake']) self.register_jvm_tool(self._jmake_bootstrap_key, external_tools) self._compiler_bootstrap_key = 'java-compiler' compiler_bootstrap_tools = self.context.config.getlist('java-compile', 'compiler-bootstrap-tools', default=['//:java-compiler']) self.register_jvm_tool(self._compiler_bootstrap_key, compiler_bootstrap_tools) self.configure_args(args_defaults=_JAVA_COMPILE_ARGS_DEFAULT, warning_defaults=_JAVA_COMPILE_WARNING_ARGS_DEFAULT, no_warning_defaults=_JAVA_COMPILE_WARNING_ARGS_DEFAULT) self._javac_opts = [] if self.get_options().args: for arg in self.get_options().args: self._javac_opts.extend(safe_shlex_split(arg)) else: self._javac_opts.extend(self.context.config.getlist('java-compile', 'javac_args', default=[]))
def _do_run_tests(self, targets, workunit): if not targets: return PythonTestResult.rc(0) sources = list(itertools.chain(*[t.sources_relative_to_buildroot() for t in targets])) if not sources: return PythonTestResult.rc(0) with self._test_runner(targets, workunit) as (pex, test_args): def run_and_analyze(resultlog_path): result = self._do_run_tests_with_args(pex, workunit, args) failed_targets = self._get_failed_targets_from_resultlogs(resultlog_path, targets) return result.with_failed_targets(failed_targets) args = [] if self._debug: args.extend(['-s']) if self.get_options().colors: args.extend(['--color', 'yes']) for options in self.get_options().options + self.get_passthru_args(): args.extend(safe_shlex_split(options)) args.extend(test_args) args.extend(sources) # The user might have already specified the resultlog option. In such case, reuse it. resultlogs = [arg.split('=', 1)[-1] for arg in args if arg.startswith('--resultlog=')] if resultlogs: return run_and_analyze(resultlogs[-1]) else: with temporary_file_path() as resultlog_path: args.append('--resultlog={0}'.format(resultlog_path)) return run_and_analyze(resultlog_path)
def __init__(self, task_exports, context): super(_Coverage, self).__init__(task_exports, context) options = task_exports.task_options self._coverage = options.coverage self._coverage_filters = options.coverage_patterns or [] self._coverage_jvm_options = [] for jvm_option in options.coverage_jvm_options: self._coverage_jvm_options.extend(safe_shlex_split(jvm_option)) self._coverage_dir = os.path.join(task_exports.workdir, 'coverage') self._coverage_instrument_dir = os.path.join(self._coverage_dir, 'classes') # TODO(ji): These may need to be transferred down to the Emma class, as the suffixes # may be emma-specific. Resolve when we also provide cobertura support. self._coverage_metadata_file = os.path.join(self._coverage_dir, 'coverage.em') self._coverage_file = os.path.join(self._coverage_dir, 'coverage.ec') self._coverage_report_console = options.coverage_console self._coverage_console_file = os.path.join(self._coverage_dir, 'coverage.txt') self._coverage_report_xml = options.coverage_xml self._coverage_xml_file = os.path.join(self._coverage_dir, 'coverage.xml') self._coverage_report_html_open = options.coverage_html_open self._coverage_report_html = self._coverage_report_html_open or options.coverage_html self._coverage_html_file = os.path.join(self._coverage_dir, 'html', 'index.html') self._coverage_force = options.coverage_force
def execute(self): binary = self.require_single_root_target() if isinstance(binary, PythonBinary): # We can't throw if binary isn't a PythonBinary, because perhaps we were called on a # jvm_binary, in which case we have to no-op and let jvm_run do its thing. # TODO(benjy): Some more elegant way to coordinate how tasks claim targets. interpreter = self.select_interpreter_for_targets(binary.closure()) with self.cached_chroot( interpreter=interpreter, pex_info=binary.pexinfo, targets=[binary], platforms=binary.platforms ) as chroot: pex = chroot.pex() self.context.release_lock() with self.context.new_workunit(name="run", labels=[WorkUnitLabel.RUN]): args = [] for arg in self.get_options().args: args.extend(safe_shlex_split(arg)) args += self.get_passthru_args() po = pex.run(blocking=False, args=args) try: result = po.wait() if result != 0: msg = "{interpreter} {entry_point} {args} ... exited non-zero ({code})".format( interpreter=interpreter.binary, entry_point=binary.entry_point, args=" ".join(args), code=result, ) raise TaskError(msg, exit_code=result) except KeyboardInterrupt: po.send_signal(signal.SIGINT) raise
def execute(self): binary = self.require_single_root_target() if isinstance(binary, PythonBinary): # We can't throw if binary isn't a PythonBinary, because perhaps we were called on a # jvm_binary, in which case we have to no-op and let jvm_run do its thing. # TODO(benjy): Use MutexTask to coordinate this. pex = self.create_pex(binary.pexinfo) args = [] for arg in self.get_options().args: args.extend(safe_shlex_split(arg)) args += self.get_passthru_args() self.context.release_lock() with self.context.new_workunit(name='run', cmd=pex.cmdline(args), labels=[WorkUnitLabel.TOOL, WorkUnitLabel.RUN]): po = pex.run(blocking=False, args=args, env=os.environ.copy()) try: result = po.wait() if result != 0: msg = '{interpreter} {entry_point} {args} ... exited non-zero ({code})'.format( interpreter=pex.interpreter.binary, entry_point=binary.entry_point, args=' '.join(args), code=result) raise TaskError(msg, exit_code=result) except KeyboardInterrupt: po.send_signal(signal.SIGINT) raise
def __init__(self, *args, **kwargs): super(ScalaRepl, self).__init__(*args, **kwargs) self.jvm_args = self.context.config.getlist('scala-repl', 'jvm_args', default=[]) if self.context.options.run_jvmargs: for arg in self.context.options.run_jvmargs: self.jvm_args.extend(safe_shlex_split(arg)) self.confs = self.context.config.getlist('scala-repl', 'confs', default=['default']) self._bootstrap_key = 'scala-repl' bootstrap_tools = self.context.config.getlist('scala-repl', 'bootstrap-tools', default=['//:scala-repl-2.9.3']) self.register_jvm_tool(self._bootstrap_key, bootstrap_tools) self.main = self.context.config.get('scala-repl', 'main') self.args = self.context.config.getlist('scala-repl', 'args', default=[]) if self.context.options.run_args: for arg in self.context.options.run_args: self.args.extend(safe_shlex_split(arg))
def __init__(self, *args, **kwargs): super(JvmTask, self).__init__(*args, **kwargs) self.jvm_options = [] for jvm_option in self.get_options().jvm_options: self.jvm_options.extend(safe_shlex_split(jvm_option)) if self.get_options().debug: debug_port = self.get_options().debug_port self.jvm_options.extend( arg.format(debug_port=debug_port) for arg in self.get_options().debug_args) self.args = [] for arg in self.get_options().args: self.args.extend(safe_shlex_split(arg)) self.confs = self.get_options().confs
def errorprone(self, target): runtime_classpaths = self.context.products.get_data('runtime_classpath') runtime_classpath = [jar for conf, jar in runtime_classpaths.get_for_targets(target.closure(bfs=True))] output_dir = os.path.join(self.workdir, target.id) safe_mkdir(output_dir) runtime_classpath.append(output_dir) # Try to run errorprone with the same java version as the target # The minimum JDK for errorprone is JDK 1.8 min_jdk_version = max(target.platform.target_level, Revision.lenient('1.8')) if min_jdk_version.components[0] == 1: max_jdk_version = Revision(min_jdk_version.components[0], min_jdk_version.components[1], '9999') else: max_jdk_version = Revision(min_jdk_version.components[0], '9999') self.set_distribution(minimum_version=min_jdk_version, maximum_version=max_jdk_version, jdk=True) jvm_options = self.get_options().jvm_options[:] if self.dist.version < Revision.lenient('9'): # For Java 8 we need to add the errorprone javac jar to the bootclasspath to # avoid the "java.lang.NoSuchFieldError: ANNOTATION_PROCESSOR_MODULE_PATH" error # See https://github.com/google/error-prone/issues/653 for more information jvm_options.extend(['-Xbootclasspath/p:{}'.format(self.tool_classpath('errorprone-javac')[0])]) args = [ '-d', output_dir, ] # Errorprone does not recognize source or target 10 yet if target.platform.source_level < Revision.lenient('10'): args.extend(['-source', str(target.platform.source_level)]) if target.platform.target_level < Revision.lenient('10'): args.extend(['-target', str(target.platform.target_level)]) errorprone_classpath_file = os.path.join(self.workdir, '{}.classpath'.format(os.path.basename(output_dir))) with open(errorprone_classpath_file, 'w') as f: f.write('-classpath ') f.write(':'.join(runtime_classpath)) args.append('@{}'.format(errorprone_classpath_file)) for opt in self.get_options().command_line_options: args.extend(safe_shlex_split(opt)) with argfile.safe_args(self.calculate_sources(target), self.get_options()) as batched_sources: args.extend(batched_sources) result = self.runjava(classpath=self.tool_classpath('errorprone'), main=self._ERRORPRONE_MAIN, jvm_options=jvm_options, args=args, workunit_name='errorprone', workunit_labels=[WorkUnitLabel.LINT]) self.context.log.debug('java {main} ... exited with result ({result})'.format( main=self._ERRORPRONE_MAIN, result=result)) return result
def get_program_args(self): """Get the program args to run this JVM with. These are the arguments passed to main() and are program-specific. """ ret = [] for arg in self.get_options().program_args: ret.extend(safe_shlex_split(arg)) return ret
def _do_run_tests(self, targets): if not targets: return PythonTestResult.rc(0) buildroot = get_buildroot() source_chroot = os.path.relpath( self.context.products.get_data(GatherSources.PYTHON_SOURCES).path(), buildroot) sources_map = {} # Path from chroot -> Path from buildroot. for t in targets: for p in t.sources_relative_to_source_root(): sources_map[os.path.join(source_chroot, p)] = os.path.join(t.target_base, p) if not sources_map: return PythonTestResult.rc(0) with self._test_runner(targets, sources_map) as (pex, test_args): # Validate that the user didn't provide any passthru args that conflict # with those we must set ourselves. for arg in self.get_passthru_args(): if arg.startswith('--junitxml') or arg.startswith('--confcutdir'): raise TaskError('Cannot pass this arg through to pytest: {}'.format(arg)) junitxml_path = self._get_junit_xml_path(targets) # N.B. the `--confcutdir` here instructs pytest to stop scanning for conftest.py files at the # top of the buildroot. This prevents conftest.py files from outside (e.g. in users home dirs) # from leaking into pants test runs. See: https://github.com/pantsbuild/pants/issues/2726 args = ['--junitxml', junitxml_path, '--confcutdir', get_buildroot(), '--continue-on-collection-errors'] if self.get_options().fail_fast: args.extend(['-x']) if self._debug: args.extend(['-s']) if self.get_options().colors: args.extend(['--color', 'yes']) for options in self.get_options().options + self.get_passthru_args(): args.extend(safe_shlex_split(options)) args.extend(test_args) args.extend(sources_map.keys()) result = self._do_run_tests_with_args(pex, args) external_junit_xml_dir = self.get_options().junit_xml_dir if external_junit_xml_dir: safe_mkdir(external_junit_xml_dir) shutil.copy(junitxml_path, external_junit_xml_dir) failed_targets = self._get_failed_targets_from_junitxml(junitxml_path, targets) def parse_error_handler(parse_error): # Simple error handler to pass to xml parsing function. raise TaskError('Error parsing xml file at {}: {}' .format(parse_error.xml_path, parse_error.cause)) all_tests_info = self.parse_test_info(junitxml_path, parse_error_handler, ['file', 'name']) for test_name, test_info in all_tests_info.items(): test_target = self._get_target_from_test(test_info, targets) self.report_all_info_for_single_test(self.options_scope, test_target, test_name, test_info) return result.with_failed_targets(failed_targets)
def execute(self): # We drive creation of setup.py distributions from the original target graph, grabbing codegen'd # sources when needed. We ignore PythonDistribution targets. def is_exported_python_target(t): return t.is_original and self.has_provides(t) and not is_local_python_dist(t) exported_python_targets = OrderedSet(t for t in self.context.target_roots if is_exported_python_target(t)) if not exported_python_targets: raise TaskError('setup-py target(s) must provide an artifact.') dist_dir = self.get_options().pants_distdir # NB: We have to create and then run in 2 steps so that we can discover all exported targets # in-play in the creation phase which then allows a tsort of these exported targets in the run # phase to ensure an exported target is, for example (--run="sdist upload"), uploaded before any # exported target that depends on it is uploaded. created = {} def create(exported_python_target): if exported_python_target not in created: self.context.log.info('Creating setup.py project for {}'.format(exported_python_target)) subject = self.derived_by_original.get(exported_python_target, exported_python_target) setup_dir, dependencies = self.create_setup_py(subject, dist_dir) created[exported_python_target] = setup_dir if self._recursive: for dep in dependencies: if is_exported_python_target(dep): create(dep) for exported_python_target in exported_python_targets: create(exported_python_target) interpreter = self.context.products.get_data(PythonInterpreter) python_dists = self.context.products.register_data(self.PYTHON_DISTS_PRODUCT, {}) for exported_python_target in reversed(sort_targets(list(created.keys()))): setup_dir = created.get(exported_python_target) if setup_dir: if not self._run: self.context.log.info('Running packager against {}'.format(setup_dir)) setup_runner = Packager(setup_dir, interpreter=interpreter) tgz_name = os.path.basename(setup_runner.sdist()) sdist_path = os.path.join(dist_dir, tgz_name) self.context.log.info('Writing {}'.format(sdist_path)) shutil.move(setup_runner.sdist(), sdist_path) safe_rmtree(setup_dir) python_dists[exported_python_target] = sdist_path else: self.context.log.info('Running {} against {}'.format(self._run, setup_dir)) split_command = safe_shlex_split(self._run) setup_runner = SetupPyRunner(setup_dir, split_command, interpreter=interpreter) installed = setup_runner.run() if not installed: raise TaskError('Install failed.') python_dists[exported_python_target] = setup_dir
def _go_install(self, target, gopath): build_flags = re.sub(r'^"|"$', '', self.get_options().build_flags) args = safe_shlex_split(build_flags) + [target.import_path] result, go_cmd = self.go_dist.execute_go_cmd( 'install', gopath=gopath, args=args, workunit_factory=self.context.new_workunit, workunit_name='install {}'.format(target.address.spec), workunit_labels=[WorkUnitLabel.COMPILER]) if result != 0: raise TaskError('{} failed with exit code {}'.format(go_cmd, result))
def __init__(self, *args, **kwargs): super(IvyResolve, self).__init__(*args, **kwargs) self._outdir = self.get_options().outdir or os.path.join(self.workdir, 'reports') self._open = self.get_options().open self._report = self._open or self.get_options().report self._args = [] for arg in self.get_options().args: self._args.extend(safe_shlex_split(arg))
def _do_run_tests(self, targets, workunit): def _extract_resultlog_filename(args): resultlogs = [arg[arg.find('=') + 1:] for arg in args if arg.startswith('--resultlog=')] if resultlogs: return resultlogs[0] else: try: return args[args.index('--resultlog') + 1] except IndexError: self.context.log.error('--resultlog specified without an argument') return None except ValueError: return None if not targets: return PythonTestResult.rc(0) rel_sources = list(itertools.chain(*[t.sources_relative_to_source_root() for t in targets])) if not rel_sources: return PythonTestResult.rc(0) source_root = self.context.products.get_data(GatherSources.PYTHON_SOURCES).path() sources = [os.path.join(source_root, p) for p in rel_sources] with self._test_runner(targets, workunit) as (pex, test_args): def run_and_analyze(resultlog_path): result = self._do_run_tests_with_args(pex, workunit, args) failed_targets = self._get_failed_targets_from_resultlogs(resultlog_path, targets) return result.with_failed_targets(failed_targets) # N.B. the `--confcutdir` here instructs pytest to stop scanning for conftest.py files at the # top of the buildroot. This prevents conftest.py files from outside (e.g. in users home dirs) # from leaking into pants test runs. See: https://github.com/pantsbuild/pants/issues/2726 args = ['--confcutdir', get_buildroot()] if self.get_options().fail_fast: args.extend(['-x']) if self._debug: args.extend(['-s']) if self.get_options().colors: args.extend(['--color', 'yes']) for options in self.get_options().options + self.get_passthru_args(): args.extend(safe_shlex_split(options)) args.extend(test_args) args.extend(sources) # The user might have already specified the resultlog option. In such case, reuse it. resultlog_arg = _extract_resultlog_filename(args) if resultlog_arg: return run_and_analyze(resultlog_arg) else: with temporary_file_path() as resultlog_path: args.insert(0, '--resultlog={0}'.format(resultlog_path)) return run_and_analyze(resultlog_path)
def __init__(self, settings): self._settings = settings options = settings.options self._context = settings.context self._coverage = options.coverage self._coverage_jvm_options = [] for jvm_option in options.coverage_jvm_options: self._coverage_jvm_options.extend(safe_shlex_split(jvm_option)) self._coverage_open = options.coverage_open self._coverage_force = options.coverage_force
def __init__(self, *args, **kwargs): super(IvyResolve, self).__init__(*args, **kwargs) self._cachedir = IvySubsystem.global_instance().get_options().cache_dir self._classpath_dir = os.path.join(self.workdir, 'mapped') self._outdir = self.get_options().outdir or os.path.join(self.workdir, 'reports') self._open = self.get_options().open self._report = self._open or self.get_options().report self._confs = None self._args = [] for arg in self.get_options().args: self._args.extend(safe_shlex_split(arg))
def _do_run_tests(self, targets, workunit): def _extract_resultlog_filename(args): resultlogs = [arg[arg.find('=') + 1:] for arg in args if arg.startswith('--resultlog=')] if resultlogs: return resultlogs[0] else: try: return args[args.index('--resultlog') + 1] except IndexError: self.context.log.error('--resultlog specified without an argument') return None except ValueError: return None if not targets: return PythonTestResult.rc(0) sources = list(itertools.chain(*[t.sources_relative_to_buildroot() for t in targets])) if not sources: return PythonTestResult.rc(0) with self._test_runner(targets, workunit) as (pex, test_args): def run_and_analyze(resultlog_path): result = self._do_run_tests_with_args(pex, workunit, args) failed_targets = self._get_failed_targets_from_resultlogs(resultlog_path, targets) return result.with_failed_targets(failed_targets) # N.B. the `--confcutdir` here instructs pytest to stop scanning for conftest.py files at the # top of the buildroot. This prevents conftest.py files from outside (e.g. in users home dirs) # from leaking into pants test runs. See: https://github.com/pantsbuild/pants/issues/2726 args = ['--confcutdir', get_buildroot()] if self._debug: args.extend(['-s']) if self.get_options().colors: args.extend(['--color', 'yes']) for options in self.get_options().options + self.get_passthru_args(): args.extend(safe_shlex_split(options)) args.extend(test_args) args.extend(sources) # The user might have already specified the resultlog option. In such case, reuse it. resultlog_arg = _extract_resultlog_filename(args) if resultlog_arg: return run_and_analyze(resultlog_arg) else: with temporary_file_path() as resultlog_path: args.insert(0, '--resultlog={0}'.format(resultlog_path)) return run_and_analyze(resultlog_path)
def execute(self): binary = self.require_single_root_target() if isinstance(binary, PythonBinary): # We can't throw if binary isn't a PythonBinary, because perhaps we were called on a # jvm_binary, in which case we have to no-op and let jvm_run do its thing. # TODO(benjy): Use MutexTask to coordinate this. interpreter = self.context.products.get_data(PythonInterpreter) with temporary_dir() as tmpdir: # Create a wrapper pex to "merge" the other pexes into via PEX_PATH. builder = PEXBuilder(tmpdir, interpreter, pex_info=binary.pexinfo) builder.freeze() pexes = [ self.context.products.get_data( ResolveRequirements.REQUIREMENTS_PEX), self.context.products.get_data( GatherSources.PYTHON_SOURCES) ] # TODO: Expose the path as a property in pex, instead of relying on # fishing it out of the cmdline. pex_path = os.pathsep.join([pex.cmdline()[1] for pex in pexes]) pex = PEX(tmpdir, interpreter) self.context.release_lock() with self.context.new_workunit(name='run', labels=[WorkUnitLabel.RUN]): args = [] for arg in self.get_options().args: args.extend(safe_shlex_split(arg)) args += self.get_passthru_args() po = pex.run(blocking=False, args=args, env={'PEX_PATH': pex_path}) try: result = po.wait() if result != 0: msg = '{interpreter} {entry_point} {args} ... exited non-zero ({code})'.format( interpreter=interpreter.binary, entry_point=binary.entry_point, args=' '.join(args), code=result) raise TaskError(msg, exit_code=result) except KeyboardInterrupt: po.send_signal(signal.SIGINT) raise
def __init__(self, *args, **kwargs): super(IvyResolve, self).__init__(*args, **kwargs) self._cachedir = IvySubsystem.global_instance().get_options().cache_dir self._classpath_dir = os.path.join(self.workdir, 'mapped') self._outdir = self.get_options().outdir or os.path.join( self.workdir, 'reports') self._open = self.get_options().open self._report = self._open or self.get_options().report self._confs = None self._args = [] for arg in self.get_options().args: self._args.extend(safe_shlex_split(arg))
def get_jvm_options(self): """Return the options to run this JVM with. These are options to the JVM itself, such as -Dfoo=bar, -Xmx=1g, -XX:-UseParallelGC and so on. Thus named because get_options() already exists (and returns this object's Pants options). """ ret = [] for opt in self.get_options().options: ret.extend(safe_shlex_split(opt)) if self.get_options().debug: debug_port = self.get_options().debug_port ret.extend(arg.format(debug_port=debug_port) for arg in self.get_options().debug_args) return ret
def _do_run_tests(self, targets, workunit): def _extract_resultlog_filename(args): resultlogs = [arg[arg.find('=') + 1:] for arg in args if arg.startswith('--resultlog=')] if resultlogs: return resultlogs[0] else: try: return args[args.index('--resultlog') + 1] except IndexError: self.context.log.error('--resultlog specified without an argument') return None except ValueError: return None if not targets: return PythonTestResult.rc(0) sources = list(itertools.chain(*[t.sources_relative_to_buildroot() for t in targets])) if not sources: return PythonTestResult.rc(0) with self._test_runner(targets, workunit) as (pex, test_args): def run_and_analyze(resultlog_path): result = self._do_run_tests_with_args(pex, workunit, args) failed_targets = self._get_failed_targets_from_resultlogs(resultlog_path, targets) return result.with_failed_targets(failed_targets) args = [] if self._debug: args.extend(['-s']) if self.get_options().colors: args.extend(['--color', 'yes']) for options in self.get_options().options + self.get_passthru_args(): args.extend(safe_shlex_split(options)) args.extend(test_args) args.extend(sources) # The user might have already specified the resultlog option. In such case, reuse it. resultlog_arg = _extract_resultlog_filename(args) if resultlog_arg: return run_and_analyze(resultlog_arg) else: with temporary_file_path() as resultlog_path: args.insert(0, '--resultlog={0}'.format(resultlog_path)) return run_and_analyze(resultlog_path)
def __init__(self, *args, **kwargs): super(IvyResolve, self).__init__(*args, **kwargs) self._cachedir = IvySubsystem.global_instance().get_options().cache_dir self._classpath_dir = os.path.join(self.workdir, 'mapped') self._outdir = self.get_options().outdir or os.path.join(self.workdir, 'reports') self._open = self.get_options().open self._report = self._open or self.get_options().report self._confs = None self._args = [] for arg in self.get_options().args: self._args.extend(safe_shlex_split(arg)) # Typically this should be a local cache only, since classpaths aren't portable. self.setup_artifact_cache()
def _do_run_tests(self, targets, workunit): if not targets: return PythonTestResult.rc(0) buildroot = get_buildroot() source_chroot = os.path.relpath( self.context.products.get_data(GatherSources.PYTHON_SOURCES).path(), buildroot) sources_map = {} # Path from chroot -> Path from buildroot. for t in targets: for p in t.sources_relative_to_source_root(): sources_map[os.path.join(source_chroot, p)] = os.path.join(t.target_base, p) if not sources_map: return PythonTestResult.rc(0) with self._test_runner(targets, sources_map, workunit) as (pex, test_args): # Validate that the user didn't provide any passthru args that conflict # with those we must set ourselves. for arg in self.get_passthru_args(): if arg.startswith('--junitxml') or arg.startswith('--confcutdir'): raise TaskError('Cannot pass this arg through to pytest: {}'.format(arg)) junitxml_path = self._get_junit_xml_path(targets) # N.B. the `--confcutdir` here instructs pytest to stop scanning for conftest.py files at the # top of the buildroot. This prevents conftest.py files from outside (e.g. in users home dirs) # from leaking into pants test runs. See: https://github.com/pantsbuild/pants/issues/2726 args = ['--junitxml', junitxml_path, '--confcutdir', get_buildroot(), '--continue-on-collection-errors'] if self.get_options().fail_fast: args.extend(['-x']) if self._debug: args.extend(['-s']) if self.get_options().colors: args.extend(['--color', 'yes']) for options in self.get_options().options + self.get_passthru_args(): args.extend(safe_shlex_split(options)) args.extend(test_args) args.extend(sources_map.keys()) result = self._do_run_tests_with_args(pex, workunit, args) external_junit_xml_dir = self.get_options().junit_xml_dir if external_junit_xml_dir: safe_mkdir(external_junit_xml_dir) shutil.copy(junitxml_path, external_junit_xml_dir) failed_targets = self._get_failed_targets_from_junitxml(junitxml_path, targets) return result.with_failed_targets(failed_targets)
def get_jvm_options(self): """Return the options to run this JVM with. These are options to the JVM itself, such as -Dfoo=bar, -Xmx=1g, -XX:-UseParallelGC and so on. Thus named because get_options() already exists (and returns this object's Pants options). """ ret = [] for opt in self.get_options().options: ret.extend(safe_shlex_split(opt)) if (self.get_options().debug or self.get_options().is_flagged('debug_port') or self.get_options().is_flagged('debug_args')): debug_port = self.get_options().debug_port ret.extend(arg.format(debug_port=debug_port) for arg in self.get_options().debug_args) return ret
def __init__(self, *args, **kwargs): super(IvyResolve, self).__init__(*args, **kwargs) self._ivy_bootstrapper = Bootstrapper.instance() self._cachedir = self._ivy_bootstrapper.ivy_cache_dir self._classpath_dir = os.path.join(self.workdir, 'mapped') self._outdir = self.get_options().outdir or os.path.join( self.workdir, 'reports') self._open = self.get_options().open self._report = self._open or self.get_options().report self._confs = None self._args = [] for arg in self.get_options().args: self._args.extend(safe_shlex_split(arg)) # Typically this should be a local cache only, since classpaths aren't portable. self.setup_artifact_cache()
def execute(self): binary = self.require_single_root_target() if isinstance(binary, PythonBinary): # We can't throw if binary isn't a PythonBinary, because perhaps we were called on a # jvm_binary, in which case we have to no-op and let jvm_run do its thing. # TODO(benjy): Use MutexTask to coordinate this. interpreter = self.context.products.get_data(PythonInterpreter) with temporary_dir() as tmpdir: # Create a wrapper pex to "merge" the other pexes into via PEX_PATH. builder = PEXBuilder(tmpdir, interpreter, pex_info=binary.pexinfo) builder.freeze() pexes = [ self.context.products.get_data(ResolveRequirements.REQUIREMENTS_PEX), self.context.products.get_data(GatherSources.PYTHON_SOURCES), ] # TODO: Expose the path as a property in pex, instead of relying on # fishing it out of the cmdline. pex_path = os.pathsep.join([pex.cmdline()[1] for pex in pexes]) pex = PEX(tmpdir, interpreter) self.context.release_lock() with self.context.new_workunit(name="run", labels=[WorkUnitLabel.RUN]): args = [] for arg in self.get_options().args: args.extend(safe_shlex_split(arg)) args += self.get_passthru_args() po = pex.run(blocking=False, args=args, env={"PEX_PATH": pex_path}) try: result = po.wait() if result != 0: msg = "{interpreter} {entry_point} {args} ... exited non-zero ({code})".format( interpreter=interpreter.binary, entry_point=binary.entry_point, args=" ".join(args), code=result, ) raise TaskError(msg, exit_code=result) except KeyboardInterrupt: po.send_signal(signal.SIGINT) raise
def __init__(self, settings, copy2=shutil.copy2, copytree=shutil.copytree, is_file=os.path.isfile, safe_md=safe_mkdir): self._settings = settings options = settings.options self._context = settings.context self._coverage = options.coverage self._coverage_jvm_options = [] for jvm_option in options.coverage_jvm_options: self._coverage_jvm_options.extend(safe_shlex_split(jvm_option)) self._coverage_open = options.coverage_open self._coverage_force = options.coverage_force # Injecting these methods to make testing cleaner. self._copy2 = copy2 self._copytree = copytree self._is_file = is_file self._safe_makedir = safe_md
def __init__(self, task_exports, context): super(_Coverage, self).__init__(task_exports, context) options = task_exports.task_options self._coverage = options.coverage self._coverage_filters = options.coverage_patterns or [] self._coverage_jvm_options = [] for jvm_option in options.coverage_jvm_options: self._coverage_jvm_options.extend(safe_shlex_split(jvm_option)) self._coverage_dir = os.path.join(task_exports.workdir, 'coverage') self._coverage_instrument_dir = os.path.join(self._coverage_dir, 'classes') # TODO(ji): These may need to be transferred down to the Emma class, as the suffixes # may be emma-specific. Resolve when we also provide cobertura support. self._coverage_metadata_file = os.path.join(self._coverage_dir, 'coverage.em') self._coverage_file = os.path.join(self._coverage_dir, 'coverage.ec') self._coverage_console_file = os.path.join(self._coverage_dir, 'coverage.txt') self._coverage_xml_file = os.path.join(self._coverage_dir, 'coverage.xml') self._coverage_html_file = os.path.join(self._coverage_dir, 'html', 'index.html') self._coverage_open = options.coverage_open self._coverage_force = options.coverage_force
def __init__(self, options, context, workdir, tool_classpath, confs, log, copy2=shutil.copy2, copytree=shutil.copytree, is_file=os.path.isfile, safe_md=safe_mkdir): self.options = options self.context = context self.workdir = workdir self.tool_classpath = tool_classpath self.confs = confs self.log = log self.coverage_dir = os.path.join(self.workdir, 'coverage') self.coverage_instrument_dir = os.path.join(self.coverage_dir, 'classes') self.coverage_console_file = os.path.join(self.coverage_dir, 'coverage.txt') self.coverage_xml_file = os.path.join(self.coverage_dir, 'coverage.xml') self.coverage_html_file = os.path.join(self.coverage_dir, 'html', 'index.html') self.coverage_jvm_options = [] for jvm_option in options.coverage_jvm_options: self.coverage_jvm_options.extend(safe_shlex_split(jvm_option)) self.coverage_open = options.coverage_open self.coverage_force = options.coverage_force # Injecting these methods to make unit testing cleaner. self.copy2 = copy2 self.copytree = copytree self.is_file = is_file self.safe_makedir = safe_md
def execute(self): binary = self.require_single_root_target() if isinstance(binary, PythonBinary): # We can't throw if binary isn't a PythonBinary, because perhaps we were called on a # jvm_binary, in which case we have to no-op and let jvm_run do its thing. # TODO(benjy): Some more elegant way to coordinate how tasks claim targets. interpreter = self.select_interpreter_for_targets( self.context.targets()) with self.temporary_pex_builder( interpreter=interpreter, pex_info=binary.pexinfo) as builder: chroot = PythonChroot(context=self.context, targets=[binary], builder=builder, platforms=binary.platforms, interpreter=interpreter) chroot.dump() builder.freeze() pex = PEX(builder.path(), interpreter=interpreter) self.context.release_lock() with self.context.new_workunit(name='run', labels=[WorkUnit.RUN]): args = [] for arg in self.get_options().args: args.extend(safe_shlex_split(arg)) args += self.get_passthru_args() po = pex.run(blocking=False, args=args) try: result = po.wait() if result != 0: msg = '{interpreter} {entry_point} {args} ... exited non-zero ({code})'.format( interpreter=interpreter.binary, entry_point=binary.entry_point, args=' '.join(args), code=result) raise TaskError(msg, exit_code=result) except KeyboardInterrupt: po.send_signal(signal.SIGINT) raise
def execute(self): def is_python_test(target): # Note that we ignore PythonTestSuite, because we'll see the PythonTests targets # it depends on anyway,so if we don't we'll end up running the tests twice. # TODO(benjy): Once we're off the 'build' command we can get rid of python_test_suite, # or make it an alias of dependencies(). return isinstance(target, PythonTests) test_targets = list(filter(is_python_test, self.context.targets())) if test_targets: self.context.release_lock() debug = self.get_options().level == 'debug' args = ['--color', 'yes'] if self.get_options().colors else [] for options in self.get_options().options + self.get_passthru_args( ): args.extend(safe_shlex_split(options)) test_builder = PythonTestBuilder(context=self.context, targets=test_targets, args=args, interpreter=self.interpreter, fast=self.get_options().fast, debug=debug) with self.context.new_workunit( name='run', labels=[WorkUnit.TOOL, WorkUnit.TEST]) as workunit: # pytest uses py.io.terminalwriter for output. That class detects the terminal # width and attempts to use all of it. However we capture and indent the console # output, leading to weird-looking line wraps. So we trick the detection code # into thinking the terminal window is narrower than it is. cols = os.environ.get('COLUMNS', 80) with environment_as(COLUMNS=str(int(cols) - 30)): stdout = workunit.output('stdout') if workunit else None stderr = workunit.output('stderr') if workunit else None if test_builder.run(stdout=stdout, stderr=stderr): raise TaskError()
def execute(self): binary = self.require_single_root_target() if not self.source_target_constraint.satisfied_by(binary): return extra_args = [] for arg in self.get_options().args: extra_args.extend(safe_shlex_split(arg)) passthru_args = self.get_passthru_args() full_argv = extra_args + [binary.script_path, '--'] + passthru_args perl6_env = self.context.products.get_data(CollectPerl6Env.Perl6Env) self.context.release_lock() try: self._perl6.invoke_perl6( full_argv, perl6_env, workunit_factory=self._run_workunit_factory) except Perl6.Perl6InvocationError as e: raise self.Perl6RunError("Error running perl 6: {}".format(e), e, exit_code=e.exit_code)
def errorprone(self, target): runtime_classpaths = self.context.products.get_data( 'runtime_classpath') runtime_classpath = [ jar for conf, jar in runtime_classpaths.get_for_targets( target.closure(bfs=True)) ] output_dir = os.path.join(self.workdir, target.id) safe_mkdir(output_dir) runtime_classpath.append(output_dir) args = [ '-classpath', ':'.join(runtime_classpath), '-d', output_dir, ] for opt in self.get_options().command_line_options: args.extend(safe_shlex_split(opt)) args.extend(self.calculate_sources(target)) result = self.runjava(classpath=self.tool_classpath('errorprone'), main=self._ERRORPRONE_MAIN, jvm_options=self.get_options().jvm_options, args=args, workunit_name='errorprone', workunit_labels=[WorkUnitLabel.LINT]) self.context.log.debug( 'java {main} ... exited with result ({result})'.format( main=self._ERRORPRONE_MAIN, result=result)) return result
def __init__(self, options, context, workdir, tool_classpath, confs, log, copy2=shutil.copy2, copytree=shutil.copytree, is_file=os.path.isfile, safe_md=safe_mkdir): self.options = options self.context = context self.workdir = workdir self.tool_classpath = tool_classpath self.confs = confs self.log = log self.coverage_dir = os.path.join(self.workdir, 'coverage') self.coverage_jvm_options = [] for jvm_option in options.coverage_jvm_options: self.coverage_jvm_options.extend(safe_shlex_split(jvm_option)) self.coverage_open = options.coverage_open self.coverage_force = options.coverage_force # Injecting these methods to make unit testing cleaner. self.copy2 = copy2 self.copytree = copytree self.is_file = is_file self.safe_makedir = safe_md
def get_rcopts(command, key): return safe_shlex_split(config.get( command, key)) if config.has_option(command, key) else []
def _run_pytest(self, fail_fast, test_targets, workdirs): if not test_targets: return PytestResult.rc(0) # Absolute path to chrooted test file -> Path to original test file relative to the buildroot. sources_map = OrderedDict() for t in test_targets: for p in t.sources_relative_to_source_root(): sources_map[os.path.join(self._source_chroot_path, p)] = os.path.join(t.target_base, p) if not sources_map: return PytestResult.rc(0) with self._test_runner(workdirs, test_targets, sources_map) as (pytest_binary, test_args, get_pytest_rootdir): # Validate that the user didn't provide any passthru args that conflict # with those we must set ourselves. for arg in self.get_passthru_args(): if arg.startswith('--junitxml') or arg.startswith( '--confcutdir'): raise TaskError( 'Cannot pass this arg through to pytest: {}'.format( arg)) junitxml_path = workdirs.junitxml_path(*test_targets) # N.B. the `--confcutdir` here instructs pytest to stop scanning for conftest.py files at the # top of the buildroot. This prevents conftest.py files from outside (e.g. in users home dirs) # from leaking into pants test runs. See: https://github.com/pantsbuild/pants/issues/2726 args = [ '-c', pytest_binary.config_path, '--junitxml', junitxml_path, '--confcutdir', get_buildroot(), '--continue-on-collection-errors' ] if fail_fast: args.extend(['-x']) if self._debug: args.extend(['-s']) if self.get_options().colors: args.extend(['--color', 'yes']) if self.get_options().options: for opt in self.get_options().options: args.extend(safe_shlex_split(opt)) args.extend(self.get_passthru_args()) args.extend(test_args) args.extend(sources_map.keys()) # We want to ensure our reporting based off junit xml is from this run so kill results from # prior runs. if os.path.exists(junitxml_path): os.unlink(junitxml_path) with self._maybe_run_in_chroot(): result = self._do_run_tests_with_args(pytest_binary.pex, args) # There was a problem prior to test execution preventing junit xml file creation so just let # the failure result bubble. if not os.path.exists(junitxml_path): return result pytest_rootdir = get_pytest_rootdir() failed_targets = self._get_failed_targets_from_junitxml( junitxml_path, test_targets, pytest_rootdir) def parse_error_handler(parse_error): # Simple error handler to pass to xml parsing function. raise TaskError('Error parsing xml file at {}: {}'.format( parse_error.xml_path, parse_error.cause)) all_tests_info = self.parse_test_info( junitxml_path, parse_error_handler, ['file', 'name', 'classname']) for test_name, test_info in all_tests_info.items(): test_target = self._get_target_from_test( test_info, test_targets, pytest_rootdir) self.report_all_info_for_single_test(self.options_scope, test_target, test_name, test_info) return result.with_failed_targets(failed_targets)
def _build_and_test_flags(self): return [ single_flag for flags_section in self.get_options().shlexed_build_and_test_flags for single_flag in safe_shlex_split(flags_section) ]
def _split_build_flags(build_flags): return safe_shlex_split(build_flags) # Visible for testing
def get_rcopts(command, key): return safe_shlex_split(config.get(command, key)) if config.has_option(command, key) else []
def execute(self): # We drive creation of setup.py distributions from the original target graph, grabbing codegen'd # sources when needed. We ignore PythonDistribution targets. def is_exported_python_target(t): return t.is_original and self.has_provides( t) and not is_local_python_dist(t) exported_python_targets = OrderedSet(t for t in self.context.target_roots if is_exported_python_target(t)) dist_dir = self.get_options().pants_distdir # NB: We have to create and then run in 2 steps so that we can discover all exported targets # in-play in the creation phase which then allows a tsort of these exported targets in the run # phase to ensure an exported target is, for example (--run="sdist upload"), uploaded before any # exported target that depends on it is uploaded. created: Dict[PythonTarget, Path] = {} def create(exported_python_target): if exported_python_target not in created: self.context.log.info( "Creating setup.py project for {}".format( exported_python_target)) subject = self.derived_by_original.get(exported_python_target, exported_python_target) setup_dir, dependencies = self.create_setup_py( subject, dist_dir) created[exported_python_target] = Path(setup_dir) if self._recursive: for dep in dependencies: if is_exported_python_target(dep): create(dep) for exported_python_target in exported_python_targets: create(exported_python_target) interpreter = self.context.products.get_data(PythonInterpreter) python_dists = self.context.products.register_data( self.PYTHON_DISTS_PRODUCT, {}) setup_runner = SetupPyRunner.Factory.create( scope=self, interpreter=interpreter, pex_file_path=os.path.join(self.workdir, self.fingerprint, "setup-py-runner.pex"), ) for exported_python_target in reversed( sort_targets(list(created.keys()))): setup_dir = created.get(exported_python_target) if setup_dir: if not self._run: self.context.log.info( "Running sdist against {}".format(setup_dir)) sdist = setup_runner.sdist(setup_dir) tgz_name = sdist.name sdist_path = os.path.join(dist_dir, tgz_name) self.context.log.info("Writing {}".format(sdist_path)) shutil.move(sdist, sdist_path) safe_rmtree(str(setup_dir)) python_dists[exported_python_target] = sdist_path else: self.context.log.info("Running {} against {}".format( self._run, setup_dir)) split_command = safe_shlex_split(self._run) try: setup_runner.run_setup_command( source_dir=setup_dir, setup_command=split_command) except SetupPyRunner.CommandFailure as e: raise TaskError(f"Install failed: {e}") python_dists[exported_python_target] = setup_dir
def _build_and_test_flags(self): return safe_shlex_split(self.get_options().build_and_test_flags)
def execute(self): # We drive creation of setup.py distributions from the original target graph, grabbing codegen'd # sources when needed. We ignore PythonDistribution targets. def is_exported_python_target(t): return t.is_original and self.has_provides( t) and not is_local_python_dist(t) exported_python_targets = OrderedSet(t for t in self.context.target_roots if is_exported_python_target(t)) if not exported_python_targets: raise TaskError('setup-py target(s) must provide an artifact.') dist_dir = self.get_options().pants_distdir # NB: We have to create and then run in 2 steps so that we can discover all exported targets # in-play in the creation phase which then allows a tsort of these exported targets in the run # phase to ensure an exported target is, for example (--run="sdist upload"), uploaded before any # exported target that depends on it is uploaded. created = {} def create(exported_python_target): if exported_python_target not in created: self.context.log.info( 'Creating setup.py project for {}'.format( exported_python_target)) subject = self.derived_by_original.get(exported_python_target, exported_python_target) setup_dir, dependencies = self.create_setup_py( subject, dist_dir) created[exported_python_target] = setup_dir if self._recursive: for dep in dependencies: if is_exported_python_target(dep): create(dep) for exported_python_target in exported_python_targets: create(exported_python_target) interpreter = self.context.products.get_data(PythonInterpreter) python_dists = self.context.products.register_data( self.PYTHON_DISTS_PRODUCT, {}) for exported_python_target in reversed( sort_targets(list(created.keys()))): setup_dir = created.get(exported_python_target) if setup_dir: if not self._run: self.context.log.info( 'Running packager against {}'.format(setup_dir)) setup_runner = Packager(setup_dir, interpreter=interpreter) tgz_name = os.path.basename(setup_runner.sdist()) sdist_path = os.path.join(dist_dir, tgz_name) self.context.log.info('Writing {}'.format(sdist_path)) shutil.move(setup_runner.sdist(), sdist_path) safe_rmtree(setup_dir) python_dists[exported_python_target] = sdist_path else: self.context.log.info('Running {} against {}'.format( self._run, setup_dir)) split_command = safe_shlex_split(self._run) setup_runner = SetupPyRunner(setup_dir, split_command, interpreter=interpreter) setup_runner.run() python_dists[exported_python_target] = setup_dir
def errorprone(self, target): runtime_classpaths = self.context.products.get_data( 'runtime_classpath') runtime_classpath = [ jar for conf, jar in runtime_classpaths.get_for_targets( target.closure(bfs=True)) ] output_dir = os.path.join(self.workdir, target.id) safe_mkdir(output_dir) runtime_classpath.append(output_dir) # Try to run errorprone with the same java version as the target # The minimum JDK for errorprone is JDK 1.8 min_jdk_version = max(target.platform.target_level, Revision.lenient('1.8')) if min_jdk_version.components[0] == 1: max_jdk_version = Revision(min_jdk_version.components[0], min_jdk_version.components[1], '9999') else: max_jdk_version = Revision(min_jdk_version.components[0], '9999') self.set_distribution(minimum_version=min_jdk_version, maximum_version=max_jdk_version, jdk=True) jvm_options = self.get_options().jvm_options[:] if self.dist.version < Revision.lenient('9'): # For Java 8 we need to add the errorprone javac jar to the bootclasspath to # avoid the "java.lang.NoSuchFieldError: ANNOTATION_PROCESSOR_MODULE_PATH" error # See https://github.com/google/error-prone/issues/653 for more information jvm_options.extend([ '-Xbootclasspath/p:{}'.format( self.tool_classpath('errorprone-javac')[0]) ]) args = [ '-d', output_dir, ] # Errorprone does not recognize source or target 10 yet if target.platform.source_level < Revision.lenient('10'): args.extend(['-source', str(target.platform.source_level)]) if target.platform.target_level < Revision.lenient('10'): args.extend(['-target', str(target.platform.target_level)]) errorprone_classpath_file = os.path.join( self.workdir, '{}.classpath'.format(os.path.basename(output_dir))) with open(errorprone_classpath_file, 'w') as f: f.write('-classpath ') f.write(':'.join(runtime_classpath)) args.append('@{}'.format(errorprone_classpath_file)) for opt in self.get_options().command_line_options: args.extend(safe_shlex_split(opt)) with argfile.safe_args(self.calculate_sources(target), self.get_options()) as batched_sources: args.extend(batched_sources) result = self.runjava(classpath=self.tool_classpath('errorprone'), main=self._ERRORPRONE_MAIN, jvm_options=jvm_options, args=args, workunit_name='errorprone', workunit_labels=[WorkUnitLabel.LINT]) self.context.log.debug( 'java {main} ... exited with result ({result})'.format( main=self._ERRORPRONE_MAIN, result=result)) return result
def _run_pytest(self, fail_fast, test_targets, workdirs): if not test_targets: return PytestResult.rc(0) # Absolute path to chrooted test file -> Path to original test file relative to the buildroot. sources_map = OrderedDict() for t in test_targets: for p in t.sources_relative_to_source_root(): sources_map[os.path.join(self._source_chroot_path, p)] = os.path.join(t.target_base, p) if not sources_map: return PytestResult.rc(0) with self._test_runner(workdirs, test_targets, sources_map) as (pytest_binary, test_args, get_pytest_rootdir): # Validate that the user didn't provide any passthru args that conflict # with those we must set ourselves. for arg in self.get_passthru_args(): if arg.startswith('--junitxml') or arg.startswith('--confcutdir'): raise TaskError('Cannot pass this arg through to pytest: {}'.format(arg)) junitxml_path = workdirs.junitxml_path(*test_targets) # N.B. the `--confcutdir` here instructs pytest to stop scanning for conftest.py files at the # top of the buildroot. This prevents conftest.py files from outside (e.g. in users home dirs) # from leaking into pants test runs. See: https://github.com/pantsbuild/pants/issues/2726 args = ['-c', pytest_binary.config_path, '--junitxml', junitxml_path, '--confcutdir', get_buildroot(), '--continue-on-collection-errors'] if fail_fast: args.extend(['-x']) if self._debug: args.extend(['-s']) if self.get_options().colors: args.extend(['--color', 'yes']) if self.get_options().options: for opt in self.get_options().options: args.extend(safe_shlex_split(opt)) args.extend(self.get_passthru_args()) args.extend(test_args) args.extend(sources_map.keys()) # We want to ensure our reporting based off junit xml is from this run so kill results from # prior runs. if os.path.exists(junitxml_path): os.unlink(junitxml_path) with self._maybe_run_in_chroot(): result = self._do_run_tests_with_args(pytest_binary.pex, args) # There was a problem prior to test execution preventing junit xml file creation so just let # the failure result bubble. if not os.path.exists(junitxml_path): return result pytest_rootdir = get_pytest_rootdir() failed_targets = self._get_failed_targets_from_junitxml(junitxml_path, test_targets, pytest_rootdir) def parse_error_handler(parse_error): # Simple error handler to pass to xml parsing function. raise TaskError('Error parsing xml file at {}: {}' .format(parse_error.xml_path, parse_error.cause)) all_tests_info = self.parse_test_info(junitxml_path, parse_error_handler, ['file', 'name', 'classname']) for test_name, test_info in all_tests_info.items(): test_target = self._get_target_from_test(test_info, test_targets, pytest_rootdir) self.report_all_info_for_single_test(self.options_scope, test_target, test_name, test_info) return result.with_failed_targets(failed_targets)