def __init__(self, target, root_dir, extra_targets=None, extra_requirements=None, builder=None, platforms=None, interpreter=None, conn_timeout=None): self._config = Config.load() self._target = target self._root = root_dir self._platforms = platforms self._interpreter = interpreter or PythonInterpreter.get() self._extra_targets = list( extra_targets) if extra_targets is not None else [] self._extra_requirements = list( extra_requirements) if extra_requirements is not None else [] self._builder = builder or PEXBuilder(tempfile.mkdtemp(), interpreter=self._interpreter) # Note: unrelated to the general pants artifact cache. self._egg_cache_root = os.path.join( PythonSetup(self._config).scratch_dir('artifact_cache', default_name='artifacts'), str(self._interpreter.identity)) self._key_generator = CacheKeyGenerator() self._build_invalidator = BuildInvalidator(self._egg_cache_root)
def __init__(self, target, run_tracker, interpreter=None, conn_timeout=None): self.target = target self.interpreter = interpreter or PythonInterpreter.get() if not isinstance(target, PythonBinary): raise PythonBinaryBuilder.NotABinaryTargetException( "Target %s is not a PythonBinary!" % target) config = Config.load() self.distdir = config.getdefault('pants_distdir') distpath = tempfile.mktemp(dir=self.distdir, prefix=target.name) run_info = run_tracker.run_info build_properties = {} build_properties.update( run_info.add_basic_info(run_id=None, timestamp=time.time())) build_properties.update(run_info.add_scm_info()) pexinfo = target.pexinfo.copy() pexinfo.build_properties = build_properties builder = PEXBuilder(distpath, pex_info=pexinfo, interpreter=self.interpreter) self.chroot = PythonChroot(targets=[target], builder=builder, platforms=target.platforms, interpreter=self.interpreter, conn_timeout=conn_timeout)
def yield_pex_builder(zip_safe=True): with nested(temporary_dir(), make_distribution('p1', zipped=True, zip_safe=zip_safe)) as (td, p1): pb = PEXBuilder(path=td) pb.add_egg(p1.location) yield pb
def __init__(self, target, root_dir, extra_targets=None, builder=None, interpreter=None, conn_timeout=None): self._config = Config.load() self._target = target self._root = root_dir self._interpreter = interpreter or PythonInterpreter.get() self._extra_targets = list( extra_targets) if extra_targets is not None else [] self._resolver = MultiResolver(self._config, target, conn_timeout=conn_timeout) self._builder = builder or PEXBuilder(tempfile.mkdtemp(), interpreter=self._interpreter) # Note: unrelated to the general pants artifact cache. self._egg_cache_root = os.path.join( self._config.get('python-setup', 'artifact_cache'), str(self._interpreter.identity)) self._key_generator = CacheKeyGenerator() self._build_invalidator = BuildInvalidator(self._egg_cache_root)
def create_binary(self, binary): if binary.compatibility: interpreter = self.select_interpreter(binary.compatibility) else: interpreter = self.interpreter distpath = tempfile.mktemp(dir=self._distdir, prefix=binary.name) run_info = self.context.run_tracker.run_info build_properties = {} build_properties.update( run_info.add_basic_info(run_id=None, timestamp=time.time())) build_properties.update(run_info.add_scm_info()) pexinfo = binary.pexinfo.copy() pexinfo.build_properties = build_properties builder = PEXBuilder(distpath, pex_info=pexinfo, interpreter=interpreter) chroot = PythonChroot(targets=[binary], builder=builder, platforms=binary.platforms, interpreter=interpreter, conn_timeout=self.conn_timeout) pex_path = os.path.join(self._distdir, '%s.pex' % binary.name) builder = chroot.dump() try: builder.build(pex_path) finally: builder.chroot().delete()
def main(argv): parser = argparse.ArgumentParser() parser.add_argument('--entry-point', default='__main__') parser.add_argument('--python', default=sys.executable) parser.add_argument('output') args = parser.parse_args(argv[1:]) # The manifest is passed via stdin, as it can sometimes get too large # to be passed as a CLA. manifest = json.load(sys.stdin) # Setup a temp dir that the PEX builder will use as its scratch dir. tmp_dir = tempfile.mkdtemp() try: # The version of pkg_resources.py (from setuptools) on some distros is # too old for PEX. So we keep a recent version in the buck repo and # force it into the process by constructing a custom PythonInterpreter # instance using it. interpreter = PythonInterpreter( args.python, PythonInterpreter.from_binary(args.python).identity, extras={('setuptools', '1.0'): os.path.join(BUCK_ROOT, 'third-party/py/setuptools')}) pex_builder = PEXBuilder( path=tmp_dir, interpreter=interpreter, ) # Mark this PEX as zip-safe, meaning everything will stayed zipped up # and we'll rely on python's zip-import mechanism to load modules from # the PEX. This may not work in some situations (e.g. native # libraries, libraries that want to find resources via the FS), so # we'll want to revisit this. pex_builder.info.zip_safe = True # Set the starting point for this PEX. pex_builder.info.entry_point = args.entry_point # Add the sources listed in the manifest. for dst, src in manifest['modules'].iteritems(): # NOTE(agallagher): calls the `add_source` and `add_resource` below # hard-link the given source into the PEX temp dir. Since OS X and # Linux behave different when hard-linking a source that is a # symbolic link (Linux does *not* follow symlinks), resolve any # layers of symlinks here to get consistent behavior. pex_builder.add_source(dereference_symlinks(src), dst) # Add resources listed in the manifest. for dst, src in manifest['resources'].iteritems(): # NOTE(agallagher): see rationale above. pex_builder.add_resource(dereference_symlinks(src), dst) # Generate the PEX file. pex_builder.build(args.output) # Always try cleaning up the scratch dir, ignoring failures. finally: shutil.rmtree(tmp_dir, True)
def write_pex(td, exe_contents): with open(os.path.join(td, 'exe.py'), 'w') as fp: fp.write(exe_contents) pb = PEXBuilder(path=td) pb.set_executable(os.path.join(td, 'exe.py')) pb.freeze() return pb
def execute(self): if self.options.pex and self.options.ipython: self.error('Cannot specify both --pex and --ipython!') if self.options.entry_point and self.options.ipython: self.error('Cannot specify both --entry_point and --ipython!') if self.options.verbose: print('Build operating on target: %s %s' % (self.target, 'Extra targets: %s' % ' '.join(map(str, self.extra_targets)) if self.extra_targets else '')) builder = PEXBuilder(tempfile.mkdtemp(), interpreter=self.interpreter, pex_info=self.target.pexinfo if isinstance(self.target, PythonBinary) else None) if self.options.entry_point: builder.set_entry_point(self.options.entry_point) if self.options.ipython: if not self.config.has_section('python-ipython'): self.error('No python-ipython sections defined in your pants.ini!') builder.info.entry_point = self.config.get('python-ipython', 'entry_point') if builder.info.entry_point is None: self.error('Must specify entry_point for IPython in the python-ipython section ' 'of your pants.ini!') requirements = self.config.getlist('python-ipython', 'requirements', default=[]) for requirement in requirements: self.extra_requirements.append(PythonRequirement(requirement)) executor = PythonChroot( self.target, self.root_dir, builder=builder, interpreter=self.interpreter, extra_targets=self.extra_targets, extra_requirements=self.extra_requirements, conn_timeout=self.options.conn_timeout) executor.dump() if self.options.pex: pex_name = os.path.join(self.root_dir, 'dist', '%s.pex' % self.target.name) builder.build(pex_name) print('Wrote %s' % pex_name) return 0 else: builder.freeze() pex = PEX(builder.path(), interpreter=self.interpreter) po = pex.run(args=list(self.args), blocking=False) try: return po.wait() except KeyboardInterrupt: po.send_signal(signal.SIGINT) raise
def build_pex(args, options): interpreter = None if options.python: if os.path.exists(options.python): interpreter = PythonInterpreter.from_binary(options.python) else: interpreter = PythonInterpreter.from_env(options.python) if interpreter is None: die('Failed to find interpreter: %s' % options.python) pex_builder = PEXBuilder( path=safe_mkdtemp(), interpreter=interpreter, ) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path fetchers = [Fetcher(options.repos)] if options.pypi: fetchers.append(PyPIFetcher()) resolveds = requirement_resolver(options.requirements, cache=options.cache_dir, fetchers=fetchers, interpreter=interpreter, platform=options.platform) if resolveds: log('Resolved distributions:', v=options.verbosity) for pkg in resolveds: log(' %s' % pkg, v=options.verbosity) pex_builder.add_distribution(pkg) pex_builder.add_requirement(pkg.as_requirement()) for source_dir in options.source_dirs: try: egg_path = EggInstaller(source_dir).bdist() except EggInstaller.Error: die('Failed to run installer for %s' % source_dir, CANNOT_DISTILL) pex_builder.add_egg(egg_path) if options.entry_point is not None: log('Setting entry point to %s' % options.entry_point, v=options.verbosity) pex_builder.info.entry_point = options.entry_point else: log('Creating environment PEX.', v=options.verbosity) return pex_builder
def write_pex(td, exe_contents, dists=None): dists = dists or [] with open(os.path.join(td, 'exe.py'), 'w') as fp: fp.write(exe_contents) pb = PEXBuilder(path=td) for dist in dists: pb.add_egg(dist.location) pb.set_executable(os.path.join(td, 'exe.py')) pb.freeze() return pb
def _run_python_test(self, target): po = None rv = PythonTestResult.exception() coverage_rc = None coverage_enabled = 'PANTS_PY_COVERAGE' in os.environ try: builder = PEXBuilder(interpreter=self.interpreter) builder.info.entry_point = target.entry_point builder.info.ignore_errors = target._soft_dependencies chroot = PythonChroot(target, self.root_dir, extra_targets=self.generate_test_targets(), builder=builder, platforms=('current', ), interpreter=self.interpreter, conn_timeout=self._conn_timeout) builder = chroot.dump() builder.freeze() test_args = PythonTestBuilder.generate_junit_args(target) test_args.extend(self.args) if coverage_enabled: coverage_rc, args = self.cov_setup(target, builder.chroot()) test_args.extend(args) sources = [ os.path.join(target.target_base, source) for source in target.sources ] po = PEX(builder.path(), interpreter=self.interpreter).run( args=test_args + sources, blocking=False, setsid=True) # TODO(wickman) If coverage is enabled, write an intermediate .html that points to # each of the coverage reports generated and webbrowser.open to that page. rv = PythonTestBuilder.wait_on(po, timeout=target.timeout) except Exception as e: import traceback print('Failed to run test!', file=sys.stderr) traceback.print_exc() rv = PythonTestResult.exception() finally: if coverage_rc: os.unlink(coverage_rc) if po and po.returncode != 0: try: os.killpg(po.pid, signal.SIGTERM) except OSError as e: if e.errno == errno.EPERM: print("Unable to kill process group: %d" % po.pid) elif e.errno != errno.ESRCH: rv = PythonTestResult.exception() self.successes[target._create_id()] = rv return rv
def build_pex(args, options): interpreter = interpreter_from_options(options) pex_builder = PEXBuilder( path=safe_mkdtemp(), interpreter=interpreter, ) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path installer = WheelInstaller if options.use_wheel else EggInstaller resolveds = requirement_resolver( options.requirements, obtainer=build_obtainer(options), interpreter=interpreter, platform=options.platform) if resolveds: log('Resolved distributions:', v=options.verbosity) for pkg in resolveds: log(' %s' % pkg, v=options.verbosity) pex_builder.add_distribution(pkg) pex_builder.add_requirement(pkg.as_requirement()) for source_dir in options.source_dirs: try: bdist = installer(source_dir).bdist() except installer.Error: die('Failed to run installer for %s' % source_dir, CANNOT_DISTILL) pex_builder.add_dist_location(bdist) if options.entry_point is not None: log('Setting entry point to %s' % options.entry_point, v=options.verbosity) pex_builder.info.entry_point = options.entry_point else: log('Creating environment PEX.', v=options.verbosity) return pex_builder
def __init__(self, target, root_dir, extra_targets=None, builder=None): self._config = Config.load() self._target = target self._root = root_dir self._cache = BuildCache( os.path.join(self._config.get('python-setup', 'artifact_cache'), '%s' % PythonIdentity.get())) self._extra_targets = list( extra_targets) if extra_targets is not None else [] self._resolver = PythonResolver([self._target] + self._extra_targets) self._builder = builder or PEXBuilder(tempfile.mkdtemp()) self._platforms = (Platform.current(), ) self._pythons = (sys.version[:3], ) # TODO(wickman) Should this be in the binary builder? if isinstance(self._target, PythonBinary): self._platforms = self._target._platforms self._pythons = self._target._interpreters
def __init__(self, target, args, root_dir): self.target = target if not isinstance(target, PythonBinary): raise PythonBinaryBuilder.NotABinaryTargetException( "Target %s is not a PythonBinary!" % target) config = Config.load() self.distdir = config.getdefault('pants_distdir') distpath = tempfile.mktemp(dir=self.distdir, prefix=target.name) self.builder = PEXBuilder(distpath) # configure builder PexInfo options for repo in target._repositories: self.builder.info().add_repository(repo) for index in target._indices: self.builder.info().add_index(index) self.builder.info().allow_pypi = target._allow_pypi self.builder.info().zip_safe = target._zip_safe self.builder.info().inherit_path = target._inherit_path self.builder.info().entry_point = target._entry_point self.builder.info().ignore_errors = target._ignore_errors self.chroot = PythonChroot(target, root_dir, builder=self.builder)
def _run_python_tests(self, targets, stdout, stderr): coverage_rc = None coverage_enabled = 'PANTS_PY_COVERAGE' in os.environ try: builder = PEXBuilder(interpreter=self.interpreter) builder.info.entry_point = 'pytest' chroot = PythonChroot( targets=targets, extra_requirements=self._TESTING_TARGETS, builder=builder, platforms=('current',), interpreter=self.interpreter, conn_timeout=self._conn_timeout) builder = chroot.dump() builder.freeze() test_args = [] test_args.extend(PythonTestBuilder.generate_junit_args(targets)) test_args.extend(self.args) if coverage_enabled: coverage_rc, args = self.cov_setup(targets) test_args.extend(args) sources = list(itertools.chain(*[t.sources_relative_to_buildroot() for t in targets])) pex = PEX(builder.path(), interpreter=self.interpreter) rc = pex.run(args=test_args + sources, blocking=True, setsid=True, stdout=stdout, stderr=stderr) # TODO(wickman): If coverage is enabled, write an intermediate .html that points to # each of the coverage reports generated and webbrowser.open to that page. rv = PythonTestResult.rc(rc) except Exception: import traceback print('Failed to run test!', file=stderr) traceback.print_exc() rv = PythonTestResult.exception() finally: if coverage_rc: os.unlink(coverage_rc) return rv
def _run_python_test(self, target): po = None rv = PythonTestResult.exception() try: builder = PEXBuilder() builder.info().entry_point = 'pytest' builder.info().ignore_errors = target._soft_dependencies chroot = PythonChroot(target, self.root_dir, extra_targets=self.generate_test_targets(), builder=builder) builder = chroot.dump() builder.freeze() test_args = PythonTestBuilder.generate_junit_args(target) test_args.extend(self.args) sources = [ os.path.join(target.target_base, source) for source in target.sources ] po = PEX(builder.path()).run(args=test_args + sources, blocking=False, setsid=True) rv = PythonTestBuilder.wait_on(po, timeout=target.timeout) except Exception as e: import traceback print('Failed to run test!', file=sys.stderr) traceback.print_exc() rv = PythonTestResult.exception() finally: if po and po.returncode != 0: try: os.killpg(po.pid, signal.SIGTERM) except OSError as e: if e.errno == errno.EPERM: print("Unable to kill process group: %d" % po.pid) elif e.errno != errno.ESRCH: rv = PythonTestResult.exception() self.successes[target._create_id()] = rv return rv
def __init__(self, target, root_dir, extra_targets=None, builder=None, conn_timeout=None): self._config = Config.load() self._target = target self._root = root_dir self._key_generator = CacheKeyGenerator() self._extra_targets = list( extra_targets) if extra_targets is not None else [] self._resolver = MultiResolver.from_target(self._config, target, conn_timeout=conn_timeout) self._builder = builder or PEXBuilder(tempfile.mkdtemp()) artifact_cache_root = os.path.join( self._config.get('python-setup', 'artifact_cache'), '%s' % PythonIdentity.get()) self._artifact_cache = FileBasedArtifactCache( None, self._root, artifact_cache_root, self._builder.add_dependency_file)
def main(): parser = optparse.OptionParser(usage="usage: %prog [options] output") parser.add_option('--entry-point', default='__main__') parser.add_option('--no-zip-safe', action='store_false', dest='zip_safe', default=True) parser.add_option('--python', default=sys.executable) options, args = parser.parse_args() if len(args) == 1: output = args[0] else: parser.error("'output' positional argument is required") return 1 # The manifest is passed via stdin, as it can sometimes get too large # to be passed as a CLA. manifest = json.load(sys.stdin) # Setup a temp dir that the PEX builder will use as its scratch dir. tmp_dir = tempfile.mkdtemp() try: # The version of pkg_resources.py (from setuptools) on some distros is # too old for PEX. So we keep a recent version in the buck repo and # force it into the process by constructing a custom PythonInterpreter # instance using it. interpreter = PythonInterpreter( options.python, PythonInterpreter.from_binary(options.python).identity, extras={}) pex_builder = PEXBuilder( path=tmp_dir, interpreter=interpreter, ) # Set whether this PEX as zip-safe, meaning everything will stayed zipped up # and we'll rely on python's zip-import mechanism to load modules from # the PEX. This may not work in some situations (e.g. native # libraries, libraries that want to find resources via the FS). pex_builder.info.zip_safe = options.zip_safe # Set the starting point for this PEX. pex_builder.info.entry_point = options.entry_point # Copy in our version of `pkg_resources`. pex_builder.add_source( dereference_symlinks(pkg_resources_py), os.path.join(pex_builder.BOOTSTRAP_DIR, 'pkg_resources.py')) # Add the sources listed in the manifest. for dst, src in manifest['modules'].iteritems(): # NOTE(agallagher): calls the `add_source` and `add_resource` below # hard-link the given source into the PEX temp dir. Since OS X and # Linux behave different when hard-linking a source that is a # symbolic link (Linux does *not* follow symlinks), resolve any # layers of symlinks here to get consistent behavior. try: pex_builder.add_source(dereference_symlinks(src), dst) except OSError as e: raise Exception("Failed to add {}: {}".format(src, e)) # Add resources listed in the manifest. for dst, src in manifest['resources'].iteritems(): # NOTE(agallagher): see rationale above. pex_builder.add_resource(dereference_symlinks(src), dst) # Add prebuilt libraries listed in the manifest. for req in manifest.get('prebuiltLibraries', []): try: pex_builder.add_dist_location(req) except Exception as e: raise Exception("Failed to add {}: {}".format(req, e)) # TODO(mikekap): Do something about manifest['nativeLibraries']. # Generate the PEX file. pex_builder.build(output) # Always try cleaning up the scratch dir, ignoring failures. finally: shutil.rmtree(tmp_dir, True)