def create_setup_py(self, target, dist_dir): chroot = Chroot(dist_dir, name=target.provides.name) dependency_calculator = self.DependencyCalculator(self.context.build_graph) reduced_deps = dependency_calculator.reduced_dependencies(target) self.write_contents(target, reduced_deps, chroot) self.write_setup(target, reduced_deps, chroot) target_base = '{}-{}'.format(target.provides.name, target.provides.version) setup_dir = os.path.join(dist_dir, target_base) safe_rmtree(setup_dir) shutil.move(chroot.path(), setup_dir) return setup_dir, reduced_deps
def execute(self): config = Config.load() distdir = config.getdefault('pants_distdir') setup_dir = os.path.join( distdir, '%s-%s' % (self.target.provides._name, self.target.provides._version)) chroot = Chroot(distdir, name=self.target.provides._name) self.write_sources(chroot) self.write_setup(chroot) if os.path.exists(setup_dir): import shutil shutil.rmtree(setup_dir) os.rename(chroot.path(), setup_dir) with pushd(setup_dir): cmd = '%s setup.py %s' % (sys.executable, self.options.run or 'sdist') print('Running "%s" in %s' % (cmd, setup_dir)) extra_args = {} if self.options.run else dict( stdout=subprocess.PIPE, stderr=subprocess.PIPE) po = subprocess.Popen(cmd, shell=True, **extra_args) po.wait() if self.options.run: print('Ran %s' % cmd) print('Output in %s' % setup_dir) return po.returncode elif po.returncode != 0: print('Failed to run %s!' % cmd) for line in po.stdout.read().splitlines(): print('stdout: %s' % line) for line in po.stderr.read().splitlines(): print('stderr: %s' % line) return po.returncode expected_tgz = '%s-%s.tar.gz' % (self.target.provides._name, self.target.provides._version) expected_target = os.path.join(setup_dir, 'dist', expected_tgz) dist_tgz = os.path.join(distdir, expected_tgz) if not os.path.exists(expected_target): print('Could not find expected target %s!' % expected_target) sys.exit(1) safe_delete(dist_tgz) os.rename(expected_target, dist_tgz) print('Wrote %s' % dist_tgz) safe_rmtree(setup_dir)
def execute(self): dist_dir = self._config.getdefault('pants_distdir') target_base = '%s-%s' % ( self.target.provides.name, self.target.provides.version) setup_dir = os.path.join(dist_dir, target_base) expected_tgz = '%s.tar.gz' % target_base expected_target = os.path.join(setup_dir, 'dist', expected_tgz) dist_tgz = os.path.join(dist_dir, expected_tgz) chroot = Chroot(dist_dir, name=self.target.provides.name) self.write_contents(chroot) self.write_setup(chroot) safe_rmtree(setup_dir) os.rename(chroot.path(), setup_dir) with pushd(setup_dir): cmd = '%s setup.py %s' % (sys.executable, self.options.run or 'sdist') print('Running "%s" in %s' % (cmd, setup_dir)) extra_args = {} if self.options.run else dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE) po = subprocess.Popen(cmd, shell=True, **extra_args) stdout, stderr = po.communicate() if self.options.run: print('Ran %s' % cmd) print('Output in %s' % setup_dir) return po.returncode elif po.returncode != 0: print('Failed to run %s!' % cmd) for line in ''.join(stdout).splitlines(): print('stdout: %s' % line) for line in ''.join(stderr).splitlines(): print('stderr: %s' % line) return po.returncode else: if not os.path.exists(expected_target): print('Could not find expected target %s!' % expected_target) sys.exit(1) safe_delete(dist_tgz) os.rename(expected_target, dist_tgz) safe_rmtree(setup_dir) print('Wrote %s' % dist_tgz)
def execute(self): config = Config.load() distdir = config.getdefault('pants_distdir') setup_dir = os.path.join(distdir, '%s-%s' % ( self.target.provides._name, self.target.provides._version)) chroot = Chroot(distdir, name=self.target.provides._name) self.write_sources(chroot) self.write_setup(chroot) if os.path.exists(setup_dir): import shutil shutil.rmtree(setup_dir) os.rename(chroot.path(), setup_dir) with pushd(setup_dir): cmd = '%s setup.py %s' % (sys.executable, self.options.run or 'sdist') print('Running "%s" in %s' % (cmd, setup_dir)) extra_args = {} if self.options.run else dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE) po = subprocess.Popen(cmd, shell=True, **extra_args) po.wait() if self.options.run: print('Ran %s' % cmd) print('Output in %s' % setup_dir) return po.returncode elif po.returncode != 0: print('Failed to run %s!' % cmd) for line in po.stdout.read().splitlines(): print('stdout: %s' % line) for line in po.stderr.read().splitlines(): print('stderr: %s' % line) return po.returncode expected_tgz = '%s-%s.tar.gz' % (self.target.provides._name, self.target.provides._version) expected_target = os.path.join(setup_dir, 'dist', expected_tgz) dist_tgz = os.path.join(distdir, expected_tgz) if not os.path.exists(expected_target): print('Could not find expected target %s!' % expected_target) sys.exit(1) safe_delete(dist_tgz) os.rename(expected_target, dist_tgz) print('Wrote %s' % dist_tgz) safe_rmtree(setup_dir)
def run_one(self, target): dist_dir = self._config.getdefault('pants_distdir') chroot = Chroot(dist_dir, name=target.provides.name) self.write_contents(target, chroot) self.write_setup(target, chroot) target_base = '%s-%s' % (target.provides.name, target.provides.version) setup_dir = os.path.join(dist_dir, target_base) safe_rmtree(setup_dir) shutil.move(chroot.path(), setup_dir) if not self.old_options.run: print('Running packager against %s' % setup_dir) setup_runner = Packager(setup_dir) tgz_name = os.path.basename(setup_runner.sdist()) print('Writing %s' % os.path.join(dist_dir, tgz_name)) shutil.move(setup_runner.sdist(), os.path.join(dist_dir, tgz_name)) safe_rmtree(setup_dir) else: print('Running %s against %s' % (self.old_options.run, setup_dir)) setup_runner = SetupPyRunner(setup_dir, self.old_options.run) setup_runner.run()
def run_one(self, target): dist_dir = self._config.getdefault('pants_distdir') chroot = Chroot(dist_dir, name=target.provides.name) self.write_contents(target, chroot) self.write_setup(target, chroot) target_base = '%s-%s' % (target.provides.name, target.provides.version) setup_dir = os.path.join(dist_dir, target_base) safe_rmtree(setup_dir) os.rename(chroot.path(), setup_dir) if not self.options.run: print('Running packager against %s' % setup_dir) setup_runner = Packager(setup_dir) tgz_name = os.path.basename(setup_runner.sdist()) print('Writing %s' % os.path.join(dist_dir, tgz_name)) os.rename(setup_runner.sdist(), os.path.join(dist_dir, tgz_name)) safe_rmtree(setup_dir) else: print('Running %s against %s' % (self.options.run, setup_dir)) setup_runner = SetupPyRunner(setup_dir, self.options.run) setup_runner.run()
def run_one(self, target): dist_dir = self.get_options().pants_distdir chroot = Chroot(dist_dir, name=target.provides.name) self.write_contents(target, chroot) self.write_setup(target, chroot) target_base = '%s-%s' % (target.provides.name, target.provides.version) setup_dir = os.path.join(dist_dir, target_base) safe_rmtree(setup_dir) shutil.move(chroot.path(), setup_dir) if not self._run: self.context.log.info('Running packager against %s' % setup_dir) setup_runner = Packager(setup_dir) tgz_name = os.path.basename(setup_runner.sdist()) self.context.log.info('Writing %s' % os.path.join(dist_dir, tgz_name)) shutil.move(setup_runner.sdist(), os.path.join(dist_dir, tgz_name)) safe_rmtree(setup_dir) else: self.context.log.info('Running %s against %s' % (self._run, setup_dir)) setup_runner = SetupPyRunner(setup_dir, self._run) setup_runner.run()
def __init__(self, path=None): self._chroot = Chroot(path or tempfile.mkdtemp()) self._pex_info = PexInfo.default() self._frozen = False self._logger = logging.getLogger(__name__)
class PEXBuilder(object): class InvalidDependency(Exception): pass class InvalidExecutableSpecification(Exception): pass DEPENDENCY_DIR = ".deps" BOOTSTRAP_DIR = ".bootstrap" def __init__(self, path=None): self._chroot = Chroot(path or tempfile.mkdtemp()) self._pex_info = PexInfo.default() self._frozen = False self._logger = logging.getLogger(__name__) def chroot(self): return self._chroot def path(self): return self.chroot().path() def info(self): return self._pex_info def add_source(self, filename, env_filename): self._chroot.link(filename, env_filename, "source") if filename.endswith('.py'): env_filename_pyc = os.path.splitext(env_filename)[0] + '.pyc' with open(filename) as fp: pyc_object = CodeMarshaller.from_py(fp.read(), env_filename) self._chroot.write(pyc_object.to_pyc(), env_filename_pyc, 'source') def add_resource(self, filename, env_filename): self._chroot.link(filename, env_filename, "resource") def add_requirement(self, req, dynamic=False, repo=None): self._pex_info.add_requirement(req, repo=repo, dynamic=dynamic) def add_dependency_file(self, filename, env_filename): # TODO(wickman) This is broken. The build cache abstraction just breaks down here. if filename.endswith('.egg'): self.add_egg(filename) else: self._chroot.link( filename, os.path.join(PEXBuilder.DEPENDENCY_DIR, env_filename)) def add_egg(self, egg): """ helper for add_distribution """ metadata = EggMetadata(zipimporter(egg)) dist = Distribution.from_filename(egg, metadata) self.add_distribution(dist) self.add_requirement(dist.as_requirement(), dynamic=False, repo=None) def add_distribution(self, dist): if not dist.location.endswith('.egg'): raise PEXBuilder.InvalidDependency( 'Non-egg dependencies not yet supported.') self._chroot.link( dist.location, os.path.join(PEXBuilder.DEPENDENCY_DIR, os.path.basename(dist.location))) def set_executable(self, filename, env_filename=None): if env_filename is None: env_filename = os.path.basename(filename) if self._chroot.get("executable"): raise PEXBuilder.InvalidExecutableSpecification( "Setting executable on a PEXBuilder that already has one!") self._chroot.link(filename, env_filename, "executable") entry_point = env_filename entry_point.replace(os.path.sep, '.') self._pex_info.entry_point = entry_point.rpartition('.')[0] def _prepare_inits(self): relative_digest = self._chroot.get("source") init_digest = set() for path in relative_digest: split_path = path.split(os.path.sep) for k in range(1, len(split_path)): sub_path = os.path.sep.join(split_path[0:k] + ['__init__.py']) if sub_path not in relative_digest and sub_path not in init_digest: self._chroot.touch(sub_path) init_digest.add(sub_path) def _prepare_manifest(self): self._chroot.write(self._pex_info.dump().encode('utf-8'), PexInfo.PATH, label='manifest') def _prepare_main(self): self._chroot.write(BOOTSTRAP_ENVIRONMENT, '__main__.py', label='main') def _prepare_bootstrap(self): """ Write enough of distribute into the .pex .bootstrap directory so that we can be fully self-contained. """ bare_env = pkg_resources.Environment() distribute_req = pkg_resources.Requirement.parse('distribute>=0.6.24') distribute_dist = None for dist in DistributionHelper.all_distributions(sys.path): if dist in distribute_req and bare_env.can_add(dist): distribute_dist = dist break else: raise DistributionNotFound('Could not find distribute!') for fn, content in DistributionHelper.walk_data(distribute_dist): if fn.startswith('pkg_resources.py') or fn.startswith( 'setuptools'): self._chroot.write(content, os.path.join(self.BOOTSTRAP_DIR, fn), 'resource') libraries = ('twitter.common.dirutil', 'twitter.common.collections', 'twitter.common.contextutil', 'twitter.common.lang', 'twitter.common.python', 'twitter.common.python.http', 'twitter.common.quantity') for name in libraries: dirname = name.replace('.', '/') provider = pkg_resources.get_provider(name) if not isinstance(provider, pkg_resources.DefaultProvider): mod = __import__(name, fromlist=['wutttt']) provider = pkg_resources.ZipProvider(mod) for fn in provider.resource_listdir(''): if fn.endswith('.py'): self._chroot.write( provider.get_resource_string(name, fn), os.path.join(self.BOOTSTRAP_DIR, dirname, fn), 'resource') for initdir in ('twitter', 'twitter/common'): self._chroot.write( b"__import__('pkg_resources').declare_namespace(__name__)", os.path.join(self.BOOTSTRAP_DIR, initdir, '__init__.py'), 'resource') def freeze(self): if self._frozen: return self._prepare_inits() self._prepare_manifest() self._prepare_bootstrap() self._prepare_main() self._frozen = True def build(self, filename): self.freeze() try: os.unlink(filename + '~') self._logger.warn( 'Previous binary unexpectedly exists, cleaning: %s' % (filename + '~')) except OSError: # The expectation is that the file does not exist, so continue pass with open(filename + '~', 'ab') as pexfile: assert os.path.getsize(pexfile.name) == 0 # TODO(wickman) Make this tunable pexfile.write( Compatibility.to_bytes('%s\n' % PythonIdentity.get().hashbang())) self._chroot.zip(filename + '~', mode='a') if os.path.exists(filename): os.unlink(filename) os.rename(filename + '~', filename) chmod_plus_x(filename)
class PEXBuilder(object): class InvalidDependency(Exception): pass class InvalidExecutableSpecification(Exception): pass DEPENDENCY_DIR = ".deps" BOOTSTRAP_DIR = ".bootstrap" def __init__(self, path=None): self._chroot = Chroot(path or tempfile.mkdtemp()) self._pex_info = PexInfo.default() self._frozen = False self._logger = logging.getLogger(__name__) def chroot(self): return self._chroot def path(self): return self.chroot().path() def info(self): return self._pex_info def add_source(self, filename, env_filename): self._chroot.link(filename, env_filename, "source") if filename.endswith('.py'): env_filename_pyc = os.path.splitext(env_filename)[0] + '.pyc' with open(filename) as fp: pyc_object = CodeMarshaller.from_py(fp.read(), env_filename) self._chroot.write(pyc_object.to_pyc(), env_filename_pyc, 'source') def add_resource(self, filename, env_filename): self._chroot.link(filename, env_filename, "resource") def add_requirement(self, req, dynamic=False, repo=None): self._pex_info.add_requirement(req, repo=repo, dynamic=dynamic) def add_dependency_file(self, filename, env_filename): # TODO(wickman) This is broken. The build cache abstraction just breaks down here. if filename.endswith('.egg'): self.add_egg(filename) else: self._chroot.link(filename, os.path.join(PEXBuilder.DEPENDENCY_DIR, env_filename)) def add_egg(self, egg): """ helper for add_distribution """ metadata = EggMetadata(zipimporter(egg)) dist = Distribution.from_filename(egg, metadata) self.add_distribution(dist) self.add_requirement(dist.as_requirement(), dynamic=False, repo=None) def add_distribution(self, dist): if not dist.location.endswith('.egg'): raise PEXBuilder.InvalidDependency('Non-egg dependencies not yet supported.') self._chroot.link(dist.location, os.path.join(PEXBuilder.DEPENDENCY_DIR, os.path.basename(dist.location))) def set_executable(self, filename, env_filename=None): if env_filename is None: env_filename = os.path.basename(filename) if self._chroot.get("executable"): raise PEXBuilder.InvalidExecutableSpecification( "Setting executable on a PEXBuilder that already has one!") self._chroot.link(filename, env_filename, "executable") entry_point = env_filename entry_point.replace(os.path.sep, '.') self._pex_info.entry_point = entry_point.rpartition('.')[0] def _prepare_inits(self): relative_digest = self._chroot.get("source") init_digest = set() for path in relative_digest: split_path = path.split(os.path.sep) for k in range(1, len(split_path)): sub_path = os.path.sep.join(split_path[0:k] + ['__init__.py']) if sub_path not in relative_digest and sub_path not in init_digest: self._chroot.touch(sub_path) init_digest.add(sub_path) def _prepare_manifest(self): self._chroot.write(self._pex_info.dump().encode('utf-8'), PexInfo.PATH, label='manifest') def _prepare_main(self): self._chroot.write(BOOTSTRAP_ENVIRONMENT, '__main__.py', label='main') def _prepare_bootstrap(self): """ Write enough of distribute into the .pex .bootstrap directory so that we can be fully self-contained. """ bare_env = pkg_resources.Environment() distribute_req = pkg_resources.Requirement.parse('distribute>=0.6.24') distribute_dist = None for dist in DistributionHelper.all_distributions(sys.path): if dist in distribute_req and bare_env.can_add(dist): distribute_dist = dist break else: raise DistributionNotFound('Could not find distribute!') for fn, content in DistributionHelper.walk_data(distribute_dist): if fn.startswith('pkg_resources.py') or fn.startswith('setuptools'): self._chroot.write(content, os.path.join(self.BOOTSTRAP_DIR, fn), 'resource') libraries = ( 'twitter.common.dirutil', 'twitter.common.collections', 'twitter.common.contextutil', 'twitter.common.lang', 'twitter.common.python', 'twitter.common.python.http', 'twitter.common.quantity' ) for name in libraries: dirname = name.replace('.', '/') provider = pkg_resources.get_provider(name) if not isinstance(provider, pkg_resources.DefaultProvider): mod = __import__(name, fromlist=['wutttt']) provider = pkg_resources.ZipProvider(mod) for fn in provider.resource_listdir(''): if fn.endswith('.py'): self._chroot.write(provider.get_resource_string(name, fn), os.path.join(self.BOOTSTRAP_DIR, dirname, fn), 'resource') for initdir in ('twitter', 'twitter/common'): self._chroot.write( b"__import__('pkg_resources').declare_namespace(__name__)", os.path.join(self.BOOTSTRAP_DIR, initdir, '__init__.py'), 'resource') def freeze(self): if self._frozen: return self._prepare_inits() self._prepare_manifest() self._prepare_bootstrap() self._prepare_main() self._frozen = True def build(self, filename): self.freeze() try: os.unlink(filename + '~') self._logger.warn('Previous binary unexpectedly exists, cleaning: %s' % (filename + '~')) except OSError: # The expectation is that the file does not exist, so continue pass with open(filename + '~', 'ab') as pexfile: assert os.path.getsize(pexfile.name) == 0 # TODO(wickman) Make this tunable pexfile.write(Compatibility.to_bytes('%s\n' % PythonIdentity.get().hashbang())) self._chroot.zip(filename + '~', mode='a') if os.path.exists(filename): os.unlink(filename) os.rename(filename + '~', filename) chmod_plus_x(filename)
def __init__(self, path): self._chroot = Chroot(path)
class PythonEnvironment(object): class InvalidDependency(Exception): pass class InvalidExecutableSpecification(Exception): pass DEPENDENCY_DIR = ".deps" MAIN = """ import os import sys from twitter.common.python import PythonLauncher __entry_point__ = None if locals().has_key('__file__') and __file__ is not None: __entry_point__ = os.path.dirname(__file__) elif locals().has_key('__loader__'): from zipimport import zipimporter from pkgutil import ImpLoader if isinstance(__loader__, zipimporter): __entry_point__ = __loader__.archive elif isinstance(__loader__, ImpLoader): __entry_point__ = os.path.dirname(__loader__.get_filename()) if __entry_point__ is not None: PythonLauncher(__entry_point__).execute() else: print >> sys.stderr, "Could not launch Python executable!" sys.exit(2) """ def __init__(self, path): self._chroot = Chroot(path) def chroot(self): return self._chroot def path(self): return self.chroot().path() def add_source(self, filename, env_filename): self._chroot.link(filename, env_filename, "source") def add_resource(self, filename, env_filename): self._chroot.link(filename, env_filename, "resource") def add_dependency(self, dependency): added_files = set() if not isinstance(dependency, PythonDependency): raise PythonEnvironment.InvalidDependency( "Input dependency (%s) is not a valid PythonDependency!" % repr(dependency)) for fn, content in dependency.files(): added_files.add(os.path.join(self.path(), PythonEnvironment.DEPENDENCY_DIR, fn)) self._chroot.write(content, os.path.join(PythonEnvironment.DEPENDENCY_DIR, fn), "dependency") return (os.path.join(self.path(), PythonEnvironment.DEPENDENCY_DIR), added_files) def add_dependency_file(self, filename, dep_filename): self._chroot.link(filename, os.path.join(PythonEnvironment.DEPENDENCY_DIR, dep_filename), "dependency") def set_executable(self, filename, env_filename=None): if env_filename is None: env_filename = os.path.basename(filename) self._chroot.link(filename, env_filename, "executable") def executable(self): """ Return the executable target of this environment if one is specified. If not specified, return None. """ exe = self._chroot.get("executable") if len(exe) > 1: raise PythonEnvironment.InvalidExecutableSpecification( "Expected one or zero executables, but instead got executables = %s" % repr(exe)) if len(exe) == 1: return exe.copy().pop() def _prepare_inits(self): relative_digest = self._chroot.get("source") init_digest = set() for path in relative_digest: split_path = path.split(os.path.sep) for k in range(1, len(split_path)): sub_path = os.path.sep.join(split_path[0:k] + ['__init__.py']) if sub_path not in relative_digest and sub_path not in init_digest: self._chroot.touch(sub_path) init_digest.add(sub_path) def _script(self): """ Returns the module-form of the entry point or None if it's not there. """ if self.executable(): source = self.executable() source.replace(os.path.sep, '.') source = source.rpartition('.')[0] return source def _manifest(self): manifest = {} script = self._script() if script: manifest.update({'entry': script}) return json.dumps(manifest) def _prepare_manifest(self): self._chroot.write(self._manifest(), 'PEX-INFO', label='manifest') def _prepare_main(self): self._chroot.write(self.MAIN, '__main__.py', label='main') def freeze(self): self._prepare_inits() self._prepare_manifest() self._prepare_main()
def __init__(self, path=None): self._chroot = Chroot(path or tempfile.mkdtemp()) self._pex_info = PexInfo.default() self._frozen = False