def execute(self): if self.options.verbose: print("Build operating on target: %s %s" % (self.target, 'Extra targets: %s' % ' '.join(map(str, self.extra_targets)) if self.extra_targets else '')) if self.options.resolve: executor = PythonResolver([self.target] + self.extra_targets) executor.dump() return 0 executor = PythonChroot(self.target, self.root_dir, extra_targets=self.extra_targets, conn_timeout=self.options.conn_timeout) builder = executor.dump() if self.options.pex: pex_name = os.path.join(self.root_dir, 'dist', '%s.pex' % self.target.name) builder.build(pex_name) print('Wrote %s' % pex_name) return 0 else: builder.freeze() pex = PEX(builder.path()) po = pex.run(args=list(self.args), blocking=False) try: po.wait() except KeyboardInterrupt: po.send_signal(signal.SIGINT) raise
def execute(self): binary = self.require_single_root_target() if isinstance(binary, PythonBinary): # We can't throw if binary isn't a PythonBinary, because perhaps we were called on a # jvm_binary, in which case we have to no-op and let jvm_run do its thing. # TODO(benjy): Some more elegant way to coordinate how tasks claim targets. interpreter = self.select_interpreter_for_targets(self.context.targets()) with self.temporary_pex_builder(interpreter=interpreter, pex_info=binary.pexinfo) as builder: chroot = PythonChroot( targets=[binary], builder=builder, platforms=binary.platforms, interpreter=interpreter, conn_timeout=self.conn_timeout) chroot.dump() builder.freeze() pex = PEX(builder.path(), interpreter=interpreter) self.context.lock.release() with self.context.new_workunit(name='run', labels=[WorkUnit.RUN]): po = pex.run(blocking=False) try: return po.wait() except KeyboardInterrupt: po.send_signal(signal.SIGINT) raise
def test_pex_builder(): # test w/ and w/o zipfile dists with nested(temporary_dir(), make_distribution('p1', zipped=True)) as (td, p1): write_pex(td, exe_main, dists=[p1]) success_txt = os.path.join(td, 'success.txt') PEX(td).run(args=[success_txt]) assert os.path.exists(success_txt) with open(success_txt) as fp: assert fp.read() == 'success' # test w/ and w/o zipfile dists with nested(temporary_dir(), temporary_dir(), make_distribution('p1', zipped=True)) as (td1, td2, p1): target_egg_dir = os.path.join(td2, os.path.basename(p1.location)) safe_mkdir(target_egg_dir) with closing(zipfile.ZipFile(p1.location, 'r')) as zf: zf.extractall(target_egg_dir) p1 = DistributionHelper.distribution_from_path(target_egg_dir) write_pex(td1, exe_main, dists=[p1]) success_txt = os.path.join(td1, 'success.txt') PEX(td1).run(args=[success_txt]) assert os.path.exists(success_txt) with open(success_txt) as fp: assert fp.read() == 'success'
def main(): parser = configure_clp() options, args = parser.parse_args() verbosity = 5 if options.verbosity else -1 with Tracer.env_override( PEX_VERBOSE=verbosity, TWITTER_COMMON_PYTHON_HTTP=verbosity, PYTHON_VERBOSE=verbosity): pex_builder = build_pex(args, options) if options.pex_name is not None: log('Saving PEX file to %s' % options.pex_name, v=options.verbosity) tmp_name = options.pex_name + '~' safe_delete(tmp_name) pex_builder.build(tmp_name) os.rename(tmp_name, options.pex_name) return 0 if options.platform != Platform.current(): log('WARNING: attempting to run PEX with differing platform!') pex_builder.freeze() log('Running PEX file at %s with args %s' % (pex_builder.path(), args), v=options.verbosity) pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter) return pex.run(args=list(args))
def execute(self): if self.options.pex and self.options.ipython: self.error('Cannot specify both --pex and --ipython!') if self.options.entry_point and self.options.ipython: self.error('Cannot specify both --entry_point and --ipython!') if self.options.verbose: print('Build operating on target: %s %s' % (self.target, 'Extra targets: %s' % ' '.join(map(str, self.extra_targets)) if self.extra_targets else '')) builder = PEXBuilder(tempfile.mkdtemp(), interpreter=self.interpreter, pex_info=self.target.pexinfo if isinstance(self.target, PythonBinary) else None) if self.options.entry_point: builder.set_entry_point(self.options.entry_point) if self.options.ipython: if not self.config.has_section('python-ipython'): self.error('No python-ipython sections defined in your pants.ini!') builder.info.entry_point = self.config.get('python-ipython', 'entry_point') if builder.info.entry_point is None: self.error('Must specify entry_point for IPython in the python-ipython section ' 'of your pants.ini!') requirements = self.config.getlist('python-ipython', 'requirements', default=[]) with ParseContext.temp(): for requirement in requirements: self.extra_targets.append(PythonRequirement(requirement)) executor = PythonChroot( self.target, self.root_dir, builder=builder, interpreter=self.interpreter, extra_targets=self.extra_targets, conn_timeout=self.options.conn_timeout) executor.dump() if self.options.pex: pex_name = os.path.join(self.root_dir, 'dist', '%s.pex' % self.target.name) builder.build(pex_name) print('Wrote %s' % pex_name) return 0 else: builder.freeze() pex = PEX(builder.path(), interpreter=self.interpreter) po = pex.run(args=list(self.args), blocking=False) try: return po.wait() except KeyboardInterrupt: po.send_signal(signal.SIGINT) raise
def execute(self): if self.options.pex and self.options.ipython: self.error('Cannot specify both --pex and --ipython!') if self.options.entry_point and self.options.ipython: self.error('Cannot specify both --entry_point and --ipython!') if self.options.verbose: print('Build operating on target: %s %s' % (self.target, 'Extra targets: %s' % ' '.join(map(str, self.extra_targets)) if self.extra_targets else '')) builder = PEXBuilder(tempfile.mkdtemp(), interpreter=self.interpreter, pex_info=self.target.pexinfo if isinstance(self.target, PythonBinary) else None) if self.options.entry_point: builder.set_entry_point(self.options.entry_point) if self.options.ipython: if not self.config.has_section('python-ipython'): self.error('No python-ipython sections defined in your pants.ini!') builder.info.entry_point = self.config.get('python-ipython', 'entry_point') if builder.info.entry_point is None: self.error('Must specify entry_point for IPython in the python-ipython section ' 'of your pants.ini!') requirements = self.config.getlist('python-ipython', 'requirements', default=[]) for requirement in requirements: self.extra_requirements.append(PythonRequirement(requirement)) executor = PythonChroot( self.target, self.root_dir, builder=builder, interpreter=self.interpreter, extra_targets=self.extra_targets, extra_requirements=self.extra_requirements, conn_timeout=self.options.conn_timeout) executor.dump() if self.options.pex: pex_name = os.path.join(self.root_dir, 'dist', '%s.pex' % self.target.name) builder.build(pex_name) print('Wrote %s' % pex_name) return 0 else: builder.freeze() pex = PEX(builder.path(), interpreter=self.interpreter) po = pex.run(args=list(self.args), blocking=False) try: return po.wait() except KeyboardInterrupt: po.send_signal(signal.SIGINT) raise
def register_build_properties(): if not HAS_PEX: return rm = RootMetrics().scope('build') try: build_properties = PEX().info.build_properties except PEX.NotFound: return for key, value in build_properties.items(): rm.register(Label(str(key), str(value)))
def _prepare_bootstrap(self): """ Write enough of distribute and pip into the .pex .bootstrap directory so that we can be fully self-contained. """ bare_env = pkg_resources.Environment() pip_req = pkg_resources.Requirement.parse('pip>=1.1') distribute_req = pkg_resources.Requirement.parse('distribute>=0.6.24') pip_dist = distribute_dist = None for dist in DistributionHelper.all_distributions(sys.path): if dist in pip_req and bare_env.can_add(dist): pip_dist = dist if dist in distribute_req and bare_env.can_add(dist): distribute_dist = dist if pip_dist and distribute_dist: break if not pip_dist: raise DistributionNotFound('Could not find pip!') if not distribute_dist: raise DistributionNotFound('Could not find distribute!') PEX.debug('Writing .bootstrap library.') for fn, content in DistributionHelper.walk_data(pip_dist): if fn.startswith('pip/'): # PEX.debug('BOOTSTRAP: Writing %s' % fn) self._chroot.write(content, os.path.join(self.BOOTSTRAP_DIR, fn), 'resource') for fn, content in DistributionHelper.walk_data(distribute_dist): if fn.startswith('pkg_resources.py') or fn.startswith('setuptools'): # PEX.debug('BOOTSTRAP: Writing %s' % fn) self._chroot.write(content, os.path.join(self.BOOTSTRAP_DIR, fn), 'resource') libraries = ( 'twitter.common.dirutil', 'twitter.common.collections', 'twitter.common.contextutil', 'twitter.common.lang', 'twitter.common.python' ) for name in libraries: dirname = name.replace('.', '/') provider = pkg_resources.get_provider(name) if not isinstance(provider, pkg_resources.DefaultProvider): mod = __import__(name, fromlist=['wutttt']) provider = pkg_resources.ZipProvider(mod) for fn in provider.resource_listdir(''): if fn.endswith('.py'): # PEX.debug('BOOTSTRAP: Writing %s' % os.path.join(dirname, fn)) self._chroot.write(provider.get_resource_string(name, fn), os.path.join(self.BOOTSTRAP_DIR, dirname, fn), 'resource') for initdir in ('twitter', 'twitter/common'): self._chroot.write( b"__import__('pkg_resources').declare_namespace(__name__)", os.path.join(self.BOOTSTRAP_DIR, initdir, '__init__.py'), 'resource')
def build_pex(args, options): pex_builder = PEXBuilder(path=safe_mkdtemp()) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path fetchers = [Fetcher(options.repos)] if options.pypi: fetchers.append(PyPIFetcher()) resolver = Resolver(cache=options.cache_dir, fetchers=fetchers, install_cache=options.cache_dir) resolveds = resolver.resolve(options.requirements) if resolveds: log('Resolved distributions:', v=options.verbosity) for pkg in resolveds: log(' %s' % pkg, v=options.verbosity) pex_builder.add_distribution(pkg) pex_builder.add_requirement(pkg.as_requirement()) for source_dir in options.source_dirs: dist = Installer(source_dir).distribution() if not dist: die('Failed to run installer for %s' % source_dir, CANNOT_DISTILL) egg_path = Distiller(dist).distill() if not egg_path: die('Failed to distill %s into egg' % dist, CANNOT_DISTILL) pex_builder.add_egg(egg_path) if options.entry_point is not None: log('Setting entry point to %s' % options.entry_point, v=options.verbosity) pex_builder.info.entry_point = options.entry_point else: log('Creating environment PEX.', v=options.verbosity) if options.pex_name is not None: log('Saving PEX file to %s' % options.pex_name, v=options.verbosity) tmp_name = options.pex_name + '~' safe_delete(tmp_name) pex_builder.build(tmp_name) os.rename(tmp_name, options.pex_name) else: pex_builder.freeze() log('Running PEX file at %s with args %s' % (pex_builder.path(), args), v=options.verbosity) pex = PEX(pex_builder.path()) return pex.run(args=list(args)) return 0
def _run_lint(self, target, args): chroot = PythonChroot(target, self.root_dir, extra_targets=[ Target.get(Address.parse(self.root_dir, '3rdparty/python:pylint'))]) builder = chroot.dump() builder.info().entry_point = 'pylint.lint' builder.freeze() interpreter_args = [ '--rcfile=%s' % os.path.join(self.root_dir, 'build-support', 'pylint', 'pylint.rc')] interpreter_args.extend(args or []) sources = OrderedSet([]) target.walk(lambda trg: sources.update( trg.sources if hasattr(trg, 'sources') and trg.sources is not None else [])) pex = PEX(builder.path()) pex.run(args=interpreter_args + list(sources), with_chroot=True)
def _run_lint(self, target, args): lint_target = Target.get(Address.parse(self.root_dir, self._opts['lint_target'])) assert lint_target, 'Could not find target %r' % self._opts['lint_target'] chroot = PythonChroot(target, self.root_dir, extra_targets=[lint_target], conn_timeout=self._conn_timeout) chroot.builder.info().ignore_errors = True builder = chroot.dump() builder.info().entry_point = self._opts['entry_point'] builder.info().run_name = 'main' builder.freeze() interpreter_args = self._opts['interpreter_args'] interpreter_args.extend(args or []) sources = OrderedSet([]) target.walk(lambda trg: sources.update( trg.sources if hasattr(trg, 'sources') and trg.sources is not None else [])) pex = PEX(builder.path()) pex.run(args=interpreter_args + list(sources), with_chroot=True)
def _run_lint(self, target, args): chroot = PythonChroot( target, self.root_dir, extra_targets=[Target.get(Address.parse(self.root_dir, "3rdparty/python:pylint"))], conn_timeout=self._conn_timeout, ) chroot.builder.info().ignore_errors = True builder = chroot.dump() builder.info().entry_point = "pylint.lint" builder.freeze() interpreter_args = ["--rcfile=%s" % os.path.join(self.root_dir, "build-support", "pylint", "pylint.rc")] interpreter_args.extend(args or []) sources = OrderedSet([]) target.walk( lambda trg: sources.update(trg.sources if hasattr(trg, "sources") and trg.sources is not None else []) ) pex = PEX(builder.path()) pex.run(args=interpreter_args + list(sources), with_chroot=True)
def execute(self): print("Build operating on target: %s %s" % (self.target, 'Extra targets: %s' % ' '.join(map(str, self.extra_targets)) if self.extra_targets else '')) if self.options.resolve: executor = PythonResolver(self.target, self.root_dir, extra_targets=self.extra_targets) executor.dump() return 0 executor = PythonChroot(self.target, self.root_dir, extra_targets=self.extra_targets) builder = executor.dump() if self.options.pex: pex_name = os.path.join(self.root_dir, 'dist', '%s.pex' % self.target.name) builder.build(pex_name) print('Wrote %s' % pex_name) return 0 else: builder.freeze() pex = PEX(builder.path()) return pex.run(args=list(self.args))
def execute(self): (accept_predicate, reject_predicate) = Target.lang_discriminator('python') targets = self.require_homogeneous_targets(accept_predicate, reject_predicate) if targets: # We can't throw if the target isn't a python target, because perhaps we were called on a # JVM target, in which case we have to no-op and let scala repl do its thing. # TODO(benjy): Some more elegant way to coordinate how tasks claim targets. interpreter = self.select_interpreter_for_targets(targets) extra_requirements = [] if self.context.options.python_repl_ipython: entry_point = self.context.config.get('python-ipython', 'entry_point', default='IPython:start_ipython') ipython_requirements = self.context.config.getlist('python-ipython', 'requirements', default=['ipython==1.0.0']) for req in ipython_requirements: extra_requirements.append(PythonRequirement(req)) else: entry_point = 'code:interact' with self.temporary_pex_builder(interpreter=interpreter) as builder: builder.set_entry_point(entry_point) chroot = PythonChroot( targets=targets, extra_requirements=extra_requirements, builder=builder, interpreter=interpreter, conn_timeout=self.conn_timeout) chroot.dump() builder.freeze() pex = PEX(builder.path(), interpreter=interpreter) self.context.lock.release() with stty_utils.preserve_stty_settings(): with self.context.new_workunit(name='run', labels=[WorkUnit.RUN]): po = pex.run(blocking=False) try: return po.wait() except KeyboardInterrupt: pass
def _run_lint(self, target, args): chroot = PythonChroot(target, self.root_dir, extra_targets=[ Target.get( Address.parse(self.root_dir, '3rdparty/python:pylint')) ]) builder = chroot.dump() builder.info().entry_point = 'pylint.lint' builder.freeze() interpreter_args = [ '--rcfile=%s' % os.path.join(self.root_dir, 'build-support', 'pylint', 'pylint.rc') ] interpreter_args.extend(args or []) sources = OrderedSet([]) target.walk(lambda trg: sources.update(trg.sources if hasattr( trg, 'sources') and trg.sources is not None else [])) pex = PEX(builder.path()) pex.run(args=interpreter_args + list(sources), with_chroot=True)
def _run_python_tests(self, targets, stdout, stderr): coverage_rc = None coverage_enabled = 'PANTS_PY_COVERAGE' in os.environ try: builder = PEXBuilder(interpreter=self.interpreter) builder.info.entry_point = 'pytest' chroot = PythonChroot( targets=targets, extra_requirements=self._TESTING_TARGETS, builder=builder, platforms=('current',), interpreter=self.interpreter, conn_timeout=self._conn_timeout) builder = chroot.dump() builder.freeze() test_args = [] test_args.extend(PythonTestBuilder.generate_junit_args(targets)) test_args.extend(self.args) if coverage_enabled: coverage_rc, args = self.cov_setup(targets) test_args.extend(args) sources = list(itertools.chain(*[t.sources_relative_to_buildroot() for t in targets])) pex = PEX(builder.path(), interpreter=self.interpreter) rc = pex.run(args=test_args + sources, blocking=True, setsid=True, stdout=stdout, stderr=stderr) # TODO(wickman): If coverage is enabled, write an intermediate .html that points to # each of the coverage reports generated and webbrowser.open to that page. rv = PythonTestResult.rc(rc) except Exception: import traceback print('Failed to run test!', file=stderr) traceback.print_exc() rv = PythonTestResult.exception() finally: if coverage_rc: os.unlink(coverage_rc) return rv
def _run_python_test(self, target): po = None rv = PythonTestResult.exception() coverage_rc = None coverage_enabled = 'PANTS_PY_COVERAGE' in os.environ try: builder = PEXBuilder(interpreter=self.interpreter) builder.info.entry_point = target.entry_point builder.info.ignore_errors = target._soft_dependencies chroot = PythonChroot(target, self.root_dir, extra_targets=self.generate_test_targets(), builder=builder, platforms=('current', ), interpreter=self.interpreter, conn_timeout=self._conn_timeout) builder = chroot.dump() builder.freeze() test_args = PythonTestBuilder.generate_junit_args(target) test_args.extend(self.args) if coverage_enabled: coverage_rc, args = self.cov_setup(target, builder.chroot()) test_args.extend(args) sources = [ os.path.join(target.target_base, source) for source in target.sources ] po = PEX(builder.path(), interpreter=self.interpreter).run( args=test_args + sources, blocking=False, setsid=True) # TODO(wickman) If coverage is enabled, write an intermediate .html that points to # each of the coverage reports generated and webbrowser.open to that page. rv = PythonTestBuilder.wait_on(po, timeout=target.timeout) except Exception as e: import traceback print('Failed to run test!', file=sys.stderr) traceback.print_exc() rv = PythonTestResult.exception() finally: if coverage_rc: os.unlink(coverage_rc) if po and po.returncode != 0: try: os.killpg(po.pid, signal.SIGTERM) except OSError as e: if e.errno == errno.EPERM: print("Unable to kill process group: %d" % po.pid) elif e.errno != errno.ESRCH: rv = PythonTestResult.exception() self.successes[target._create_id()] = rv return rv
def _run_python_test(self, target): po = None rv = PythonTestResult.exception() try: builder = PEXBuilder() builder.info().entry_point = 'pytest' builder.info().ignore_errors = target._soft_dependencies chroot = PythonChroot(target, self.root_dir, extra_targets=self.generate_test_targets(), builder=builder) builder = chroot.dump() builder.freeze() test_args = PythonTestBuilder.generate_junit_args(target) test_args.extend(self.args) sources = [ os.path.join(target.target_base, source) for source in target.sources ] po = PEX(builder.path()).run(args=test_args + sources, blocking=False, setsid=True) rv = PythonTestBuilder.wait_on(po, timeout=target.timeout) except Exception as e: import traceback print('Failed to run test!', file=sys.stderr) traceback.print_exc() rv = PythonTestResult.exception() finally: if po and po.returncode != 0: try: os.killpg(po.pid, signal.SIGTERM) except OSError as e: if e.errno == errno.EPERM: print("Unable to kill process group: %d" % po.pid) elif e.errno != errno.ESRCH: rv = PythonTestResult.exception() self.successes[target._create_id()] = rv return rv