def __init__(self, target, run_tracker, interpreter=None, conn_timeout=None): self.target = target self.interpreter = interpreter or PythonInterpreter.get() if not isinstance(target, PythonBinary): raise PythonBinaryBuilder.NotABinaryTargetException( "Target %s is not a PythonBinary!" % target) config = Config.load() self.distdir = config.getdefault('pants_distdir') distpath = tempfile.mktemp(dir=self.distdir, prefix=target.name) run_info = run_tracker.run_info build_properties = {} build_properties.update( run_info.add_basic_info(run_id=None, timestamp=time.time())) build_properties.update(run_info.add_scm_info()) pexinfo = target.pexinfo.copy() pexinfo.build_properties = build_properties builder = PEXBuilder(distpath, pex_info=pexinfo, interpreter=self.interpreter) self.chroot = PythonChroot(targets=[target], builder=builder, platforms=target.platforms, interpreter=self.interpreter, conn_timeout=conn_timeout)
def temporary_chroot(self, interpreter=None, pex_info=None, targets=None, extra_requirements=None, platforms=None, pre_freeze=None): """Yields a temporary PythonChroot created with the specified args. pre_freeze is an optional function run on the chroot just before freezing its builder, to allow for any extra modification. """ path = tempfile.mkdtemp() builder = PEXBuilder(path=path, interpreter=interpreter, pex_info=pex_info) with self.context.new_workunit('chroot'): chroot = PythonChroot( context=self.context, python_setup=PythonSetup.global_instance(), python_repos=PythonRepos.global_instance(), targets=targets, extra_requirements=extra_requirements, builder=builder, platforms=platforms, interpreter=interpreter) chroot.dump() if pre_freeze: pre_freeze(chroot) builder.freeze() yield chroot chroot.delete()
def execute(self): binary = self.require_single_root_target() if isinstance(binary, PythonBinary): # We can't throw if binary isn't a PythonBinary, because perhaps we were called on a # jvm_binary, in which case we have to no-op and let jvm_run do its thing. # TODO(benjy): Some more elegant way to coordinate how tasks claim targets. interpreter = self.select_interpreter_for_targets(self.context.targets()) with self.temporary_pex_builder(interpreter=interpreter, pex_info=binary.pexinfo) as builder: chroot = PythonChroot( targets=[binary], builder=builder, platforms=binary.platforms, interpreter=interpreter, conn_timeout=self.conn_timeout) chroot.dump() builder.freeze() pex = PEX(builder.path(), interpreter=interpreter) self.context.release_lock() with self.context.new_workunit(name='run', labels=[WorkUnit.RUN]): args = self.get_options().args + self.get_passthru_args() po = pex.run(blocking=False, args=args) try: result = po.wait() if result != 0: raise TaskError('python {args} ... exited non-zero ({code})' % dict(args=args, code=result), exit_code=result) except KeyboardInterrupt: po.send_signal(signal.SIGINT) raise
def _test_runner(self, targets, workunit): interpreter = self.select_interpreter_for_targets(targets) builder = PEXBuilder(interpreter=interpreter) builder.info.entry_point = 'pytest' chroot = PythonChroot( context=self.context, python_setup=PythonSetup.global_instance(), python_repos=PythonRepos.global_instance(), targets=targets, extra_requirements=self._TESTING_TARGETS, builder=builder, platforms=('current',), interpreter=interpreter) try: builder = chroot.dump() builder.freeze() pex = PEX(builder.path(), interpreter=interpreter) with self._maybe_shard() as shard_args: with self._maybe_emit_junit_xml(targets) as junit_args: with self._maybe_emit_coverage_data(targets, builder.path(), pex, workunit) as coverage_args: yield pex, shard_args + junit_args + coverage_args finally: chroot.delete()
def execute(self): binary = self.require_single_root_target() if isinstance(binary, PythonBinary): # We can't throw if binary isn't a PythonBinary, because perhaps we were called on a # jvm_binary, in which case we have to no-op and let jvm_run do its thing. # TODO(benjy): Some more elegant way to coordinate how tasks claim targets. interpreter = self.select_interpreter_for_targets( self.context.targets()) with self.temporary_pex_builder( interpreter=interpreter, pex_info=binary.pexinfo) as builder: chroot = PythonChroot(targets=[binary], builder=builder, platforms=binary.platforms, interpreter=interpreter, conn_timeout=self.conn_timeout) chroot.dump() builder.freeze() pex = PEX(builder.path(), interpreter=interpreter) self.context.lock.release() with self.context.new_workunit(name='run', labels=[WorkUnit.RUN]): po = pex.run(blocking=False) try: return po.wait() except KeyboardInterrupt: po.send_signal(signal.SIGINT) raise
def execute(self): binary = self.require_single_root_target() if isinstance(binary, PythonBinary): # We can't throw if binary isn't a PythonBinary, because perhaps we were called on a # jvm_binary, in which case we have to no-op and let jvm_run do its thing. # TODO(benjy): Some more elegant way to coordinate how tasks claim targets. interpreter = self.select_interpreter_for_targets( self.context.targets()) with self.temporary_pex_builder( interpreter=interpreter, pex_info=binary.pexinfo) as builder: chroot = PythonChroot(targets=[binary], builder=builder, platforms=binary.platforms, interpreter=interpreter) chroot.dump() builder.freeze() pex = PEX(builder.path(), interpreter=interpreter) self.context.release_lock() with self.context.new_workunit(name='run', labels=[WorkUnit.RUN]): args = self.get_options().args + self.get_passthru_args() po = pex.run(blocking=False, args=args) try: result = po.wait() if result != 0: raise TaskError( 'python {args} ... exited non-zero ({code})' % dict(args=args, code=result), exit_code=result) except KeyboardInterrupt: po.send_signal(signal.SIGINT) raise
def _build_chroot(self, path, interpreter, pex_info, targets, platforms, extra_requirements=None, executable_file_content=None): """Create a PythonChroot with the specified args.""" builder = PEXBuilder(path=path, interpreter=interpreter, pex_info=pex_info) with self.context.new_workunit('chroot'): chroot = PythonChroot(context=self.context, python_setup=PythonSetup.global_instance(), python_repos=PythonRepos.global_instance(), interpreter=interpreter, builder=builder, targets=targets, platforms=platforms, extra_requirements=extra_requirements) chroot.dump() if executable_file_content is not None: with open( os.path.join( path, '{}.py'.format(self.CHROOT_EXECUTABLE_NAME)), 'w') as outfile: outfile.write(executable_file_content) # Override any user-specified entry point, under the assumption that the # executable_file_content does what the user intends (including, probably, calling that # underlying entry point). pex_info.entry_point = self.CHROOT_EXECUTABLE_NAME builder.freeze() return chroot
def dumped_chroot(self, targets): python_repos = create_subsystem(PythonRepos) with subsystem_instance(IvySubsystem) as ivy_subsystem: ivy_bootstrapper = Bootstrapper(ivy_subsystem=ivy_subsystem) with subsystem_instance(ThriftBinary.Factory) as thrift_binary_factory: interpreter_cache = PythonInterpreterCache(self.python_setup, python_repos) interpreter_cache.setup() interpreters = list(interpreter_cache.matched_interpreters([ self.python_setup.interpreter_requirement])) self.assertGreater(len(interpreters), 0) interpreter = interpreters[0] with temporary_dir() as chroot: pex_builder = PEXBuilder(path=chroot, interpreter=interpreter) python_chroot = PythonChroot(python_setup=self.python_setup, python_repos=python_repos, ivy_bootstrapper=ivy_bootstrapper, thrift_binary_factory=thrift_binary_factory.create, interpreter=interpreter, builder=pex_builder, targets=targets, platforms=['current']) try: python_chroot.dump() yield pex_builder, python_chroot finally: python_chroot.delete()
def create_binary(self, binary): if binary.compatibility: interpreter = self.select_interpreter(binary.compatibility) else: interpreter = self.interpreter distpath = tempfile.mktemp(dir=self._distdir, prefix=binary.name) run_info = self.context.run_tracker.run_info build_properties = {} build_properties.update( run_info.add_basic_info(run_id=None, timestamp=time.time())) build_properties.update(run_info.add_scm_info()) pexinfo = binary.pexinfo.copy() pexinfo.build_properties = build_properties builder = PEXBuilder(distpath, pex_info=pexinfo, interpreter=interpreter) chroot = PythonChroot(targets=[binary], builder=builder, platforms=binary.platforms, interpreter=interpreter, conn_timeout=self.conn_timeout) pex_path = os.path.join(self._distdir, '%s.pex' % binary.name) builder = chroot.dump() try: builder.build(pex_path) finally: builder.chroot().delete()
def dumped_chroot(self, targets): # TODO(benjy): We shouldn't need to mention DistributionLocator here, as IvySubsystem # declares it as a dependency. However if we don't then test_antlr() below fails on # uninitialized options for that subsystem. Hopefully my pending (as of 9/2016) change # to clean up how we initialize and create instances of subsystems in tests will make # this problem go away. self.context(for_subsystems=[PythonRepos, PythonSetup, IvySubsystem, DistributionLocator, ThriftBinary.Factory, BinaryUtil.Factory]) python_repos = PythonRepos.global_instance() ivy_bootstrapper = Bootstrapper(ivy_subsystem=IvySubsystem.global_instance()) thrift_binary_factory = ThriftBinary.Factory.global_instance().create interpreter_cache = PythonInterpreterCache(self.python_setup, python_repos) interpreter_cache.setup() interpreters = list(interpreter_cache.matched_interpreters( self.python_setup.interpreter_constraints)) self.assertGreater(len(interpreters), 0) interpreter = interpreters[0] with temporary_dir() as chroot: pex_builder = PEXBuilder(path=chroot, interpreter=interpreter) python_chroot = PythonChroot(python_setup=self.python_setup, python_repos=python_repos, ivy_bootstrapper=ivy_bootstrapper, thrift_binary_factory=thrift_binary_factory, interpreter=interpreter, builder=pex_builder, targets=targets, platforms=['current']) try: python_chroot.dump() yield pex_builder, python_chroot finally: python_chroot.delete()
def execute(self): binary = self.require_single_root_target() if isinstance(binary, PythonBinary): # We can't throw if binary isn't a PythonBinary, because perhaps we were called on a # jvm_binary, in which case we have to no-op and let jvm_run do its thing. # TODO(benjy): Some more elegant way to coordinate how tasks claim targets. interpreter = self.select_interpreter_for_targets(self.context.targets()) with self.temporary_pex_builder(interpreter=interpreter, pex_info=binary.pexinfo) as builder: chroot = PythonChroot( targets=[binary], builder=builder, platforms=binary.platforms, interpreter=interpreter, conn_timeout=self.conn_timeout) chroot.dump() builder.freeze() pex = PEX(builder.path(), interpreter=interpreter) self.context.lock.release() with self.context.new_workunit(name='run', labels=[WorkUnit.RUN]): po = pex.run(blocking=False) try: return po.wait() except KeyboardInterrupt: po.send_signal(signal.SIGINT) raise
def execute(self): if self.old_options.pex and self.old_options.ipython: self.error('Cannot specify both --pex and --ipython!') if self.old_options.entry_point and self.old_options.ipython: self.error('Cannot specify both --entry_point and --ipython!') if self.old_options.verbose: print('Build operating on targets: %s' % ' '.join(str(target) for target in self.targets)) builder = PEXBuilder(tempfile.mkdtemp(), interpreter=self.interpreter, pex_info=self.binary.pexinfo if self.binary else None) if self.old_options.entry_point: builder.set_entry_point(self.old_options.entry_point) if self.old_options.ipython: if not self.config.has_section('python-ipython'): self.error('No python-ipython sections defined in your pants.ini!') builder.info.entry_point = self.config.get('python-ipython', 'entry_point') if builder.info.entry_point is None: self.error('Must specify entry_point for IPython in the python-ipython section ' 'of your pants.ini!') requirements = self.config.getlist('python-ipython', 'requirements', default=[]) for requirement in requirements: self.extra_requirements.append(PythonRequirement(requirement)) executor = PythonChroot( targets=self.targets, extra_requirements=self.extra_requirements, builder=builder, platforms=self.binary.platforms if self.binary else None, interpreter=self.interpreter, conn_timeout=self.old_options.conn_timeout) executor.dump() if self.old_options.pex: pex_name = self.binary.name if self.binary else Target.maybe_readable_identify(self.targets) pex_path = os.path.join(self.root_dir, 'dist', '%s.pex' % pex_name) builder.build(pex_path) print('Wrote %s' % pex_path) return 0 else: builder.freeze() pex = PEX(builder.path(), interpreter=self.interpreter) po = pex.run(args=list(self.args), blocking=False) try: return po.wait() except KeyboardInterrupt: po.send_signal(signal.SIGINT) raise
def __init__(self, target, run_tracker, interpreter=None): self.target = target self.interpreter = interpreter or PythonInterpreter.get() if not isinstance(target, PythonBinary): raise PythonBinaryBuilder.NotABinaryTargetException( "Target %s is not a PythonBinary!" % target) config = Config.from_cache() self.distdir = config.getdefault('pants_distdir') distpath = tempfile.mktemp(dir=self.distdir, prefix=target.name) run_info = run_tracker.run_info build_properties = {} build_properties.update(run_info.add_basic_info(run_id=None, timestamp=time.time())) build_properties.update(run_info.add_scm_info()) pexinfo = target.pexinfo.copy() pexinfo.build_properties = build_properties builder = PEXBuilder(distpath, pex_info=pexinfo, interpreter=self.interpreter) self.chroot = PythonChroot( targets=[target], builder=builder, platforms=target.platforms, interpreter=self.interpreter)
def execute(self): (accept_predicate, reject_predicate) = Target.lang_discriminator('python') targets = self.require_homogeneous_targets(accept_predicate, reject_predicate) if targets: # We can't throw if the target isn't a python target, because perhaps we were called on a # JVM target, in which case we have to no-op and let scala repl do its thing. # TODO(benjy): Some more elegant way to coordinate how tasks claim targets. interpreter = self.select_interpreter_for_targets(targets) extra_requirements = [] if self.context.options.python_repl_ipython: entry_point = self.context.config.get( 'python-ipython', 'entry_point', default='IPython:start_ipython') ipython_requirements = self.context.config.getlist( 'python-ipython', 'requirements', default=['ipython==1.0.0']) for req in ipython_requirements: extra_requirements.append(PythonRequirement(req)) else: entry_point = 'code:interact' with self.temporary_pex_builder( interpreter=interpreter) as builder: builder.set_entry_point(entry_point) chroot = PythonChroot(targets=targets, extra_requirements=extra_requirements, builder=builder, interpreter=interpreter, conn_timeout=self.conn_timeout) chroot.dump() builder.freeze() pex = PEX(builder.path(), interpreter=interpreter) self.context.lock.release() with stty_utils.preserve_stty_settings(): with self.context.new_workunit(name='run', labels=[WorkUnit.RUN]): po = pex.run(blocking=False) try: return po.wait() except KeyboardInterrupt: pass
def _run_python_tests(self, targets, stdout, stderr): coverage_rc = None coverage_enabled = 'PANTS_PY_COVERAGE' in os.environ try: builder = PEXBuilder(interpreter=self.interpreter) builder.info.entry_point = 'pytest' chroot = PythonChroot(targets=targets, extra_requirements=self._TESTING_TARGETS, builder=builder, platforms=('current', ), interpreter=self.interpreter, conn_timeout=self._conn_timeout) builder = chroot.dump() builder.freeze() test_args = [] test_args.extend(PythonTestBuilder.generate_junit_args(targets)) test_args.extend(self.args) if coverage_enabled: coverage_rc, args = self.cov_setup(targets) test_args.extend(args) sources = list( itertools.chain( *[t.sources_relative_to_buildroot() for t in targets])) pex = PEX(builder.path(), interpreter=self.interpreter) rc = pex.run(args=test_args + sources, blocking=True, setsid=True, stdout=stdout, stderr=stderr) # TODO(wickman): If coverage is enabled, write an intermediate .html that points to # each of the coverage reports generated and webbrowser.open to that page. rv = PythonTestResult.rc(rc) except Exception: import traceback print('Failed to run test!', file=stderr) traceback.print_exc() rv = PythonTestResult.exception() finally: if coverage_rc: os.unlink(coverage_rc) return rv
def _test_runner(self, targets, stdout, stderr): builder = PEXBuilder(interpreter=self._interpreter) builder.info.entry_point = 'pytest' chroot = PythonChroot(targets=targets, extra_requirements=self._TESTING_TARGETS, builder=builder, platforms=('current', ), interpreter=self._interpreter) try: builder = chroot.dump() builder.freeze() pex = PEX(builder.path(), interpreter=self._interpreter) with self._maybe_emit_junit_xml(targets) as junit_args: with self._maybe_emit_coverage_data(targets, builder.path(), pex, stdout, stderr) as coverage_args: yield pex, junit_args + coverage_args finally: chroot.delete()
def create_chroot(self, interpreter, builder, targets, platforms, extra_requirements): return PythonChroot(python_setup=PythonSetup.global_instance(), python_repos=PythonRepos.global_instance(), ivy_bootstrapper=self.ivy_bootstrapper, thrift_binary_factory=self.thrift_binary_factory, interpreter=interpreter, builder=builder, targets=targets, platforms=platforms, extra_requirements=extra_requirements)
def create_binary(self, binary): interpreter = self.select_interpreter_for_targets(binary.closure()) run_info = self.context.run_tracker.run_info build_properties = {} build_properties.update(run_info.add_basic_info(run_id=None, timestamp=time.time())) build_properties.update(run_info.add_scm_info()) pexinfo = binary.pexinfo.copy() pexinfo.build_properties = build_properties with self.temporary_pex_builder(pex_info=pexinfo, interpreter=interpreter) as builder: chroot = PythonChroot( targets=[binary], builder=builder, platforms=binary.platforms, interpreter=interpreter) pex_path = os.path.join(self._distdir, '%s.pex' % binary.name) chroot.dump() builder.build(pex_path)
def execute(self): (accept_predicate, reject_predicate) = Target.lang_discriminator('python') targets = self.require_homogeneous_targets(accept_predicate, reject_predicate) if targets: # We can't throw if the target isn't a python target, because perhaps we were called on a # JVM target, in which case we have to no-op and let scala repl do its thing. # TODO(benjy): Some more elegant way to coordinate how tasks claim targets. interpreter = self.select_interpreter_for_targets(targets) extra_requirements = [] if self.get_options().ipython: entry_point = self.context.config.get('python-ipython', 'entry_point', default='IPython:start_ipython') ipython_requirements = self.context.config.getlist('python-ipython', 'requirements', default=['ipython==1.0.0']) for req in ipython_requirements: extra_requirements.append(PythonRequirement(req)) else: entry_point = 'code:interact' with self.temporary_pex_builder(interpreter=interpreter) as builder: builder.set_entry_point(entry_point) chroot = PythonChroot( targets=targets, extra_requirements=extra_requirements, builder=builder, interpreter=interpreter, conn_timeout=self.conn_timeout) chroot.dump() builder.freeze() pex = PEX(builder.path(), interpreter=interpreter) self.context.lock.release() with stty_utils.preserve_stty_settings(): with self.context.new_workunit(name='run', labels=[WorkUnit.RUN]): po = pex.run(blocking=False) try: return po.wait() except KeyboardInterrupt: pass
def dumped_chroot(self, targets): python_repos = create_subsystem(PythonRepos) with subsystem_instance(IvySubsystem) as ivy_subsystem: ivy_bootstrapper = Bootstrapper(ivy_subsystem=ivy_subsystem) with subsystem_instance( ThriftBinary.Factory) as thrift_binary_factory: interpreter_cache = PythonInterpreterCache( self.python_setup, python_repos) interpreter_cache.setup() interpreters = list( interpreter_cache.matches( [self.python_setup.interpreter_requirement])) self.assertGreater(len(interpreters), 0) interpreter = interpreters[0] with temporary_dir() as chroot: pex_builder = PEXBuilder(path=chroot, interpreter=interpreter) python_chroot = PythonChroot( python_setup=self.python_setup, python_repos=python_repos, ivy_bootstrapper=ivy_bootstrapper, thrift_binary_factory=thrift_binary_factory.create, interpreter=interpreter, builder=pex_builder, targets=targets, platforms=['current']) try: python_chroot.dump() yield pex_builder, python_chroot finally: python_chroot.delete()
def dumped_chroot(self, targets): # TODO(benjy): We shouldn't need to mention DistributionLocator here, as IvySubsystem # declares it as a dependency. However if we don't then test_antlr() below fails on # uninitialized options for that subsystem. Hopefully my pending (as of 9/2016) change # to clean up how we initialize and create instances of subsystems in tests will make # this problem go away. self.context(for_subsystems=[PythonRepos, PythonSetup, IvySubsystem, DistributionLocator, ThriftBinary.Factory, BinaryUtil.Factory]) python_repos = PythonRepos.global_instance() ivy_bootstrapper = Bootstrapper(ivy_subsystem=IvySubsystem.global_instance()) thrift_binary_factory = ThriftBinary.Factory.global_instance().create interpreter_cache = PythonInterpreterCache(self.python_setup, python_repos) interpreter = interpreter_cache.select_interpreter_for_targets(targets) self.assertIsNotNone(interpreter) with temporary_dir() as chroot: pex_builder = PEXBuilder(path=chroot, interpreter=interpreter) python_chroot = PythonChroot(python_setup=self.python_setup, python_repos=python_repos, ivy_bootstrapper=ivy_bootstrapper, thrift_binary_factory=thrift_binary_factory, interpreter=interpreter, builder=pex_builder, targets=targets, platforms=['current']) try: python_chroot.dump() yield pex_builder, python_chroot finally: python_chroot.delete()
def _run_python_tests(self, targets, stdout, stderr): coverage_rc = None coverage_enabled = 'PANTS_PY_COVERAGE' in os.environ try: builder = PEXBuilder(interpreter=self.interpreter) builder.info.entry_point = 'pytest' chroot = PythonChroot( targets=targets, extra_requirements=self._TESTING_TARGETS, builder=builder, platforms=('current',), interpreter=self.interpreter, conn_timeout=self._conn_timeout) builder = chroot.dump() builder.freeze() test_args = [] test_args.extend(PythonTestBuilder.generate_junit_args(targets)) test_args.extend(self.args) if coverage_enabled: coverage_rc, args = self.cov_setup(targets) test_args.extend(args) sources = list(itertools.chain(*[t.sources_relative_to_buildroot() for t in targets])) pex = PEX(builder.path(), interpreter=self.interpreter) rc = pex.run(args=test_args + sources, blocking=True, setsid=True, stdout=stdout, stderr=stderr) # TODO(wickman): If coverage is enabled, write an intermediate .html that points to # each of the coverage reports generated and webbrowser.open to that page. rv = PythonTestResult.rc(rc) except Exception: import traceback print('Failed to run test!', file=stderr) traceback.print_exc() rv = PythonTestResult.exception() finally: if coverage_rc: os.unlink(coverage_rc) return rv
def _test_runner(self, targets, stdout, stderr): builder = PEXBuilder(interpreter=self._interpreter) builder.info.entry_point = 'pytest' chroot = PythonChroot( targets=targets, extra_requirements=self._TESTING_TARGETS, builder=builder, platforms=('current',), interpreter=self._interpreter) try: builder = chroot.dump() builder.freeze() pex = PEX(builder.path(), interpreter=self._interpreter) with self._maybe_emit_junit_xml(targets) as junit_args: with self._maybe_emit_coverage_data(targets, builder.path(), pex, stdout, stderr) as coverage_args: yield pex, junit_args + coverage_args finally: chroot.delete()
def create_binary(self, binary): interpreter = self.select_interpreter_for_targets(binary.closure()) run_info = self.context.run_tracker.run_info build_properties = {} build_properties.update(run_info.add_basic_info(run_id=None, timestamp=time.time())) build_properties.update(run_info.add_scm_info()) pexinfo = binary.pexinfo.copy() pexinfo.build_properties = build_properties with self.temporary_pex_builder(pex_info=pexinfo, interpreter=interpreter) as builder: chroot = PythonChroot( context=self.context, targets=[binary], builder=builder, platforms=binary.platforms, interpreter=interpreter) pex_path = os.path.join(self._distdir, '%s.pex' % binary.name) chroot.dump() builder.build(pex_path)
def cached_chroot(self, interpreter, pex_info, targets, platforms, extra_requirements=None, executable_file_content=None): """Returns a cached PythonChroot created with the specified args. The returned chroot will be cached for future use. TODO: Garbage-collect old chroots, so they don't pile up? TODO: Ideally chroots would just be products produced by some other task. But that's a bit too complicated to implement right now, as we'd need a way to request chroots for a variety of sets of targets. """ # This PexInfo contains any customizations specified by the caller. # The process of building a pex modifies it further. pex_info = pex_info or PexInfo.default() path = self._chroot_path(PythonSetup.global_instance(), interpreter, pex_info, targets, platforms, extra_requirements, executable_file_content) if not os.path.exists(path): path_tmp = path + '.tmp' self._build_chroot(path_tmp, interpreter, pex_info, targets, platforms, extra_requirements, executable_file_content) shutil.move(path_tmp, path) # We must read the PexInfo that was frozen into the pex, so we get the modifications # created when that pex was built. pex_info = PexInfo.from_pex(path) # Now create a PythonChroot wrapper without dumping it. builder = PEXBuilder(path=path, interpreter=interpreter, pex_info=pex_info) chroot = PythonChroot(context=self.context, python_setup=PythonSetup.global_instance(), python_repos=PythonRepos.global_instance(), interpreter=interpreter, builder=builder, targets=targets, platforms=platforms, extra_requirements=extra_requirements) # TODO: Doesn't really need to be a contextmanager, but it's convenient to make it so # while transitioning calls to temporary_chroot to calls to cached_chroot. # We can revisit after that transition is complete. yield chroot
def _build_chroot(self, path, interpreter, pex_info, targets, platforms, extra_requirements=None, executable_file_content=None): """Create a PythonChroot with the specified args.""" builder = PEXBuilder(path=path, interpreter=interpreter, pex_info=pex_info) with self.context.new_workunit('chroot'): chroot = PythonChroot( context=self.context, python_setup=PythonSetup.global_instance(), python_repos=PythonRepos.global_instance(), interpreter=interpreter, builder=builder, targets=targets, platforms=platforms, extra_requirements=extra_requirements) chroot.dump() if executable_file_content is not None: with open(os.path.join(path, '{}.py'.format(self.CHROOT_EXECUTABLE_NAME)), 'w') as outfile: outfile.write(executable_file_content) # Override any user-specified entry point, under the assumption that the # executable_file_content does what the user intends (including, probably, calling that # underlying entry point). pex_info.entry_point = self.CHROOT_EXECUTABLE_NAME builder.freeze() return chroot
class PythonBinaryBuilder(object): class NotABinaryTargetException(Exception): pass def __init__(self, target, run_tracker, interpreter=None, conn_timeout=None): self.target = target self.interpreter = interpreter or PythonInterpreter.get() if not isinstance(target, PythonBinary): raise PythonBinaryBuilder.NotABinaryTargetException( "Target %s is not a PythonBinary!" % target) config = Config.load() self.distdir = config.getdefault('pants_distdir') distpath = tempfile.mktemp(dir=self.distdir, prefix=target.name) run_info = run_tracker.run_info build_properties = {} build_properties.update( run_info.add_basic_info(run_id=None, timestamp=time.time())) build_properties.update(run_info.add_scm_info()) pexinfo = target.pexinfo.copy() pexinfo.build_properties = build_properties builder = PEXBuilder(distpath, pex_info=pexinfo, interpreter=self.interpreter) self.chroot = PythonChroot(targets=[target], builder=builder, platforms=target.platforms, interpreter=self.interpreter, conn_timeout=conn_timeout) def run(self): print('Building PythonBinary %s:' % self.target) env = self.chroot.dump() filename = os.path.join(self.distdir, '%s.pex' % self.target.name) env.build(filename) print('Wrote %s' % filename) return 0
class PythonBinaryBuilder(object): class NotABinaryTargetException(Exception): pass def __init__(self, context, target, run_tracker, interpreter=None): self.context = context self.target = target self.interpreter = interpreter or PythonInterpreter.get() if not isinstance(target, PythonBinary): raise PythonBinaryBuilder.NotABinaryTargetException( "Target %s is not a PythonBinary!" % target) config = Config.from_cache() self.distdir = config.getdefault('pants_distdir') distpath = tempfile.mktemp(dir=self.distdir, prefix=target.name) run_info = run_tracker.run_info build_properties = {} build_properties.update(run_info.add_basic_info(run_id=None, timestamp=time.time())) build_properties.update(run_info.add_scm_info()) pexinfo = target.pexinfo.copy() pexinfo.build_properties = build_properties builder = PEXBuilder(distpath, pex_info=pexinfo, interpreter=self.interpreter) self.chroot = PythonChroot( context=self.context, targets=[target], builder=builder, platforms=target.platforms, interpreter=self.interpreter) def run(self): print('Building PythonBinary %s:' % self.target) env = self.chroot.dump() filename = os.path.join(self.distdir, '%s.pex' % self.target.name) env.build(filename) print('Wrote %s' % filename) return 0
def test_get_current_platform(): expected_platforms = [Platform.current(), "linux-x86_64"] assert set(expected_platforms) == set(PythonChroot.get_platforms(["current", "linux-x86_64"]))
def execute(self): if self.options.pex and self.options.ipython: self.error('Cannot specify both --pex and --ipython!') if self.options.entry_point and self.options.ipython: self.error('Cannot specify both --entry_point and --ipython!') if self.options.verbose: print('Build operating on targets: %s' % ' '.join(str(target) for target in self.targets)) builder = PEXBuilder( tempfile.mkdtemp(), interpreter=self.interpreter, pex_info=self.binary.pexinfo if self.binary else None) if self.options.entry_point: builder.set_entry_point(self.options.entry_point) if self.options.ipython: if not self.config.has_section('python-ipython'): self.error( 'No python-ipython sections defined in your pants.ini!') builder.info.entry_point = self.config.get('python-ipython', 'entry_point') if builder.info.entry_point is None: self.error( 'Must specify entry_point for IPython in the python-ipython section ' 'of your pants.ini!') requirements = self.config.getlist('python-ipython', 'requirements', default=[]) for requirement in requirements: self.extra_requirements.append(PythonRequirement(requirement)) executor = PythonChroot( targets=self.targets, extra_requirements=self.extra_requirements, builder=builder, platforms=self.binary.platforms if self.binary else None, interpreter=self.interpreter, conn_timeout=self.options.conn_timeout) executor.dump() if self.options.pex: pex_name = self.binary.name if self.binary else Target.maybe_readable_identify( self.targets) pex_path = os.path.join(self.root_dir, 'dist', '%s.pex' % pex_name) builder.build(pex_path) print('Wrote %s' % pex_path) return 0 else: builder.freeze() pex = PEX(builder.path(), interpreter=self.interpreter) po = pex.run(args=list(self.args), blocking=False) try: return po.wait() except KeyboardInterrupt: po.send_signal(signal.SIGINT) raise
def _compile_target(self, target): # "Compiles" a target by forming an isolated chroot of its sources and transitive deps and then # attempting to import each of the target's sources in the case of a python library or else the # entry point in the case of a python binary. # # For a library with sources lib/core.py and lib/util.py a "compiler" main file would look like: # # if __name__ == '__main__': # import lib.core # import lib.util # # For a binary with entry point lib.bin:main the "compiler" main file would look like: # # if __name__ == '__main__': # from lib.bin import main # # In either case the main file is executed within the target chroot to reveal missing BUILD # dependencies. with self.context.new_workunit(name=target.address.spec): modules = [] if isinstance(target, PythonBinary): source = 'entry_point {}'.format(target.entry_point) components = target.entry_point.rsplit(':', 1) module = components[0] if len(components) == 2: function = components[1] data = TemplateData(source=source, import_statement='from {} import {}'.format(module, function)) else: data = TemplateData(source=source, import_statement='import {}'.format(module)) modules.append(data) else: for path in target.sources_relative_to_source_root(): if path.endswith('.py'): if os.path.basename(path) == '__init__.py': module_path = os.path.dirname(path) else: module_path, _ = os.path.splitext(path) source = 'file {}'.format(os.path.join(target.target_base, path)) module = module_path.replace(os.path.sep, '.') data = TemplateData(source=source, import_statement='import {}'.format(module)) modules.append(data) if not modules: # Nothing to eval, so a trivial compile success. return 0 interpreter = self.select_interpreter_for_targets([target]) if isinstance(target, PythonBinary): pexinfo, platforms = target.pexinfo, target.platforms else: pexinfo, platforms = None, None with self.temporary_pex_builder(interpreter=interpreter, pex_info=pexinfo) as builder: with self.context.new_workunit(name='resolve'): chroot = PythonChroot( context=self.context, targets=[target], builder=builder, platforms=platforms, interpreter=interpreter) chroot.dump() with temporary_file() as imports_file: generator = Generator(pkgutil.get_data(__name__, self._EVAL_TEMPLATE_PATH), chroot=chroot.path(), modules=modules) generator.write(imports_file) imports_file.close() builder.set_executable(imports_file.name, '__pants_python_eval__.py') builder.freeze() pex = PEX(builder.path(), interpreter=interpreter) with self.context.new_workunit(name='eval', labels=[WorkUnit.COMPILER, WorkUnit.RUN, WorkUnit.TOOL], cmd=' '.join(pex.cmdline())) as workunit: returncode = pex.run(stdout=workunit.output('stdout'), stderr=workunit.output('stderr')) workunit.set_outcome(WorkUnit.SUCCESS if returncode == 0 else WorkUnit.FAILURE) if returncode != 0: self.context.log.error('Failed to eval {}'.format(target.address.spec)) return returncode
def test_get_current_platform(): expected_platforms = [Platform.current(), 'linux-x86_64'] assert set(expected_platforms) == set( PythonChroot.get_platforms(['current', 'linux-x86_64']))
def _compile_target(self, target): # "Compiles" a target by forming an isolated chroot of its sources and transitive deps and then # attempting to import each of the target's sources in the case of a python library or else the # entry point in the case of a python binary. # # For a library with sources lib/core.py and lib/util.py a "compiler" main file would look like: # # if __name__ == '__main__': # import lib.core # import lib.util # # For a binary with entry point lib.bin:main the "compiler" main file would look like: # # if __name__ == '__main__': # from lib.bin import main # # In either case the main file is executed within the target chroot to reveal missing BUILD # dependencies. with self.context.new_workunit(name=target.address.spec): modules = [] if isinstance(target, PythonBinary): source = 'entry_point {}'.format(target.entry_point) components = target.entry_point.rsplit(':', 1) module = components[0] if len(components) == 2: function = components[1] data = TemplateData( source=source, import_statement='from {} import {}'.format( module, function)) else: data = TemplateData( source=source, import_statement='import {}'.format(module)) modules.append(data) else: for path in target.sources_relative_to_source_root(): if path.endswith('.py'): if os.path.basename(path) == '__init__.py': module_path = os.path.dirname(path) else: module_path, _ = os.path.splitext(path) source = 'file {}'.format( os.path.join(target.target_base, path)) module = module_path.replace(os.path.sep, '.') data = TemplateData( source=source, import_statement='import {}'.format(module)) modules.append(data) if not modules: # Nothing to eval, so a trivial compile success. return 0 interpreter = self.select_interpreter_for_targets([target]) if isinstance(target, PythonBinary): pexinfo, platforms = target.pexinfo, target.platforms else: pexinfo, platforms = None, None with self.temporary_pex_builder(interpreter=interpreter, pex_info=pexinfo) as builder: with self.context.new_workunit(name='resolve'): chroot = PythonChroot(context=self.context, targets=[target], builder=builder, platforms=platforms, interpreter=interpreter) chroot.dump() with temporary_file() as imports_file: generator = Generator(pkgutil.get_data( __name__, self._EVAL_TEMPLATE_PATH), chroot=chroot.path(), modules=modules) generator.write(imports_file) imports_file.close() builder.set_executable(imports_file.name, '__pants_python_eval__.py') builder.freeze() pex = PEX(builder.path(), interpreter=interpreter) with self.context.new_workunit( name='eval', labels=[ WorkUnit.COMPILER, WorkUnit.RUN, WorkUnit.TOOL ], cmd=' '.join(pex.cmdline())) as workunit: returncode = pex.run(stdout=workunit.output('stdout'), stderr=workunit.output('stderr')) workunit.set_outcome(WorkUnit.SUCCESS if returncode == 0 else WorkUnit.FAILURE) if returncode != 0: self.context.log.error('Failed to eval {}'.format( target.address.spec)) return returncode
def test_get_current_platform(): expected_platforms = [Platform.current(), 'linux-x86_64'] assert set(expected_platforms) == set(PythonChroot.get_platforms(['current', 'linux-x86_64']))
def test_get_current_platform(self): expected_platforms = [Platform.current(), 'linux-x86_64'] self.assertEqual(set(expected_platforms), set(PythonChroot.get_platforms(['current', 'linux-x86_64'])))
def test_get_current_platform(self): expected_platforms = [Platform.current(), 'linux-x86_64'] self.assertEqual( set(expected_platforms), set(PythonChroot.get_platforms(['current', 'linux-x86_64'])))