def __init__(self, target, root_dir): self.target = target self.root = root_dir distdir = os.path.join(self.root, 'dist') self.chroot = Chroot(root_dir, distdir, target.name) codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.') self.codegen_root = os.path.relpath(codegen_root, self.chroot.path()) self.detected_packages = set() self.detected_namespace_packages = set()
def __init__(self, target, root_dir): self.target = target self.root = root_dir distdir = os.path.join(self.root, 'dist') self.chroot = Chroot(root_dir, distdir, target.name) codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.') self.codegen_root = os.path.relpath(codegen_root, self.chroot.path()) self.detected_packages = set()
def __init__(self, target, root_dir): self.target = target self.root = root_dir distdir = os.path.join(root_dir, 'dist') self.chroot = Chroot(root_dir, distdir, target.name)
class PythonChroot(object): class InvalidDependencyException(Exception): def __init__(self, target): Exception.__init__(self, "Not a valid Python dependency! Found: %s" % target) class BuildFailureException(Exception): def __init__(self, target): Exception.__init__(self, "Not a valid Python dependency! Found: %s" % target) def __init__(self, target, root_dir): self.target = target self.root = root_dir distdir = os.path.join(root_dir, 'dist') self.chroot = Chroot(root_dir, distdir, target.name) def __del__(self): shutil.rmtree(self.chroot.path()) def path(self): return self.chroot.path() def _dump_library(self, library): def translate_module(module): if module is None: module='' return module.replace('.', os.path.sep) def copy_to_chroot(base, path, relative_to=None, label=None): src = os.path.join(base, path) dst = os.path.join(translate_module(relative_to), path) self.chroot.copy(src, dst, label) template = library._create_template_data() print ' Dumping library: %s [relative module: %s]' % (library, template.module) for filename in template.sources: copy_to_chroot(template.template_base, filename, template.module, 'sources') for filename in template.resources: copy_to_chroot(template.template_base, filename, template.module, 'resources') def _dump_inits(self): # iterate through self.digest and find missing __init__.py's relative_digest = self.chroot.get('sources') init_digest = set() for path in relative_digest: split_path = path.split(os.path.sep) for k in range(1, len(split_path)): sub_path = os.path.sep.join(split_path[0:k] + ['__init__.py']) if sub_path not in relative_digest and sub_path not in init_digest: print ' Dumping __init__: %s' % sub_path self.chroot.touch(sub_path) init_digest.add(sub_path) def _dump_egg(self, egg): print ' Dumping egg: %s' % egg for egg_path in egg.eggs: src = egg_path dst = os.path.join('.deps', os.path.basename(src)) self.chroot.copy(src, dst, 'resources') def _dump_setuptools(self): SETUPTOOLS = 'setuptools-0.6c11-py2.6.egg' print ' Dumping setuptools: %s' % SETUPTOOLS data = pkgutil.get_data(__name__, os.path.join('bootstrap', SETUPTOOLS)) dst = os.path.join('.deps', SETUPTOOLS) self.chroot.write(data, dst, 'resources') def _dump_bin(self, binary_name, base): src = os.path.join(base, binary_name) print ' Dumping binary: %s' % binary_name self.chroot.copy(src, '__main__.py', 'sources') def _dump_thrift_library(self, library): builder = PythonThriftBuilder(library, self.root) print ' Generating %s...' % library egg_file = builder.build_egg() if egg_file: egg_file = os.path.relpath(egg_file, self.root) for pkg in builder.packages(): print ' found namespace: %s' % pkg # make a random string to disambiguate possibly similarly-named eggs? randstr = ''.join(map(chr, random.sample(range(ord('a'), ord('z')), 8))) + '_' print ' copying...', self.chroot.copy(egg_file, os.path.join('.deps', randstr + os.path.basename(egg_file)), 'resources') print 'done.' else: print ' Failed!' raise PythonChroot.BuildFailureException( "Failed to build %s!" % library) def build_dep_tree(self, target): libraries = set() eggs = set() binaries = set() thrifts = set() def add_dep(trg): if isinstance(trg, PythonLibrary): if trg.sources: libraries.add(trg) for egg in [dep for dep in trg.dependencies if isinstance(dep, PythonEgg)]: eggs.add(egg) elif isinstance(trg, PythonEgg): eggs.add(trg) elif isinstance(trg, PythonBinary): binaries.add(trg) elif isinstance(trg, PythonThriftLibrary): thrifts.add(trg) elif isinstance(trg, PythonTests): # do not dump test sources/resources, but dump their # dependencies. pass else: raise PythonChroot.InvalidDependencyException(trg) return [dep for dep in trg.dependencies if not isinstance(dep, PythonEgg)] target.walk(lambda t: add_dep(t), lambda typ: not isinstance(typ, PythonEgg)) return libraries, eggs, binaries, thrifts def dump(self): print 'Building PythonBinary %s:' % self.target libraries, eggs, binaries, thrifts = self.build_dep_tree(self.target) for lib in libraries: self._dump_library(lib) self._dump_inits() if eggs: self._dump_setuptools() for egg in eggs: self._dump_egg(egg) for thr in thrifts: self._dump_thrift_library(thr) if len(binaries) > 1: print >> sys.stderr, 'WARNING: Target has multiple python_binary targets!' for binary in binaries: self._dump_bin(binary.sources[0], binary.target_base) return self.chroot
class PythonAntlrBuilder(object): """ Antlr builder. """ class CodeGenerationException(Exception): pass def __init__(self, target, root_dir): self.target = target self.root = root_dir self.module = target.module distdir = os.path.join(self.root, 'dist') self.chroot = Chroot(root_dir, distdir, target.name) codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.') self.codegen_root = os.path.relpath(codegen_root, self.chroot.path()) self.created_packages = set() self.created_namespace_packages = set() def __del__(self): self.cleanup() def packages(self): return self.created_packages def cleanup(self): shutil.rmtree(self.chroot.path()) def run_antlrs(self, output_dir): args = [ 'java', '-jar', os.path.join(self.root, 'build-support/ivy/lib/ivy-2.2.0.jar'), '-settings', os.path.join(self.root, 'build-support/ivy/ivysettings.xml'), '-dependency', 'org.antlr', 'antlr', self.target.antlr_version, '-types', 'jar', '-main', 'org.antlr.Tool', '--', '-fo', output_dir] for source in self.target.sources: abs_path = os.path.abspath(os.path.join(self.root, self.target.target_base, source)) args.append(abs_path) cwd = os.getcwd() os.chdir(self.chroot.path()) print 'PythonAntlrBuilder executing: %s' % ' '.join(args) try: po = subprocess.Popen(args) finally: os.chdir(cwd) rv = po.wait() if rv != 0: comm = po.communicate() print >> sys.stderr, 'ANTLR generation failed!' print >> sys.stderr, 'STDOUT' print >> sys.stderr, comm[0] print >> sys.stderr, 'STDERR' print >> sys.stderr, comm[1] return rv == 0 @staticmethod def path_to_module(path): return path.replace(os.path.sep, '.') def build_egg(self): gen_root = os.path.join(self.chroot.path(), self.codegen_root) # Create the package structure. path = gen_root package = '' for module_name in self.module.split('.'): path = os.path.join(path, module_name) if package == '': package = module_name else: package = package + '.' + module_name os.mkdir(path) with open(os.path.join(path, '__init__.py'), 'w') as f: if package != self.module: # Only write this in the non-leaf modules. f.write("__import__('pkg_resources').declare_namespace(__name__)") self.created_namespace_packages.add(package) self.created_packages.add(package) # autogenerate the python files that we bundle up self.run_antlrs(path) def dump_setup_py(packages, namespace_packages): boilerplate = """ from setuptools import setup setup(name = "%(target_name)s", version = "dev", description = "autogenerated ANTLR parsers for %(target_name)s", package_dir = { "": "." }, packages = %(packages)s, namespace_packages = %(namespace_packages)s) """ boilerplate = boilerplate % { 'target_name': self.target.name, 'packages': repr(packages), 'namespace_packages': repr(list(namespace_packages)) } self.chroot.write(boilerplate, os.path.join(self.codegen_root, 'setup.py')) dump_setup_py(self.created_packages, self.created_namespace_packages) egg_root = os.path.join(self.chroot.path(), self.codegen_root) egg_path = EggBuilder().build_egg(egg_root, self.target) return egg_path
class PythonThriftBuilder(object): """ Thrift builder. """ class UnknownPlatformException(Exception): def __init__(self, platform): Exception.__init__(self, "Unknown platform: %s!" % str(platform)) class CodeGenerationException(Exception): pass def __init__(self, target, root_dir): self.target = target self.root = root_dir distdir = os.path.join(self.root, 'dist') self.chroot = Chroot(root_dir, distdir, target.name) codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.') self.codegen_root = os.path.relpath(codegen_root, self.chroot.path()) self.detected_packages = set() self.detected_namespace_packages = set() def __del__(self): self.cleanup() def packages(self): return self.detected_packages def cleanup(self): shutil.rmtree(self.chroot.path()) @staticmethod def thrift_binary(root_dir): # TODO(Brian Wickman): Will we ever need to make this smarter? PLATMAP = { ('darwin', 'i386'): ['mac', '10.6', '0.5.0', 'thrift'], ('darwin', 'x86_64'): ['mac', '10.6', '0.5.0', 'thrift'], ('darwin', 'x86_64'): ['mac', '10.7', '0.5.0', 'thrift'], ('linux', 'x86_64'): ['linux', 'x86_64', '0.5.0', 'thrift'], ('linux', 'i686'): ['linux', 'i386', '0.5.0', 'thrift'] } uname = os.uname() platform = (uname[0].lower(), uname[4]) if platform not in PLATMAP: raise PythonThriftBuilder.UnknownPlatformException(platform) return os.path.join(root_dir, 'build-support', 'bin', 'thrift', *PLATMAP[platform]) def run_thrifts(self): for src in self.target.sources: if not self._run_thrift(src): raise PythonThriftBuilder.CodeGenerationException( "Could not generate .py from %s!" % src) def _run_thrift(self, source): thrift_file = source thrift_abs_path = os.path.join(self.root, self.target.target_base, thrift_file) thrift_abs_path = os.path.abspath(thrift_abs_path) args = [ PythonThriftBuilder.thrift_binary(self.root), '--gen', 'py:new_style', '-o', self.codegen_root, thrift_abs_path ] cwd = os.getcwd() os.chdir(self.chroot.path()) print 'PythonThriftBuilder executing: %s' % ' '.join(args) try: po = subprocess.Popen(args) finally: os.chdir(cwd) rv = po.wait() if rv != 0: comm = po.communicate() print >> sys.stderr, 'thrift generation failed!' print >> sys.stderr, 'STDOUT' print >> sys.stderr, comm[0] print >> sys.stderr, 'STDERR' print >> sys.stderr, comm[1] return rv == 0 @staticmethod def path_to_module(path): return path.replace(os.path.sep, '.') def build_egg(self): # autogenerate the python files that we bundle up self.run_thrifts() genpy_root = os.path.join(self.chroot.path(), self.codegen_root, 'gen-py') for dir, subdirs, files in os.walk(os.path.normpath(genpy_root)): reldir = os.path.relpath(dir, genpy_root) if reldir == '.': continue if '__init__.py' not in files: continue init_py_abspath = os.path.join(dir, '__init__.py') module_path = self.path_to_module(reldir) self.detected_packages.add(module_path) # A namespace package is one that is just a container for other # modules and subpackages. Setting their __init__.py files as follows # allows them to be distributed across multiple eggs. Without this you # couldn't have this egg share any package prefix with any other module # in any other egg or in the source tree. # # Note that the thrift compiler should always generate empty __init__.py # files, but we test for this anyway, just in case that changes. if len(files) == 1 and os.path.getsize(init_py_abspath) == 0: with open(init_py_abspath, 'w') as f: f.write( "__import__('pkg_resources').declare_namespace(__name__)" ) self.detected_namespace_packages.add(module_path) if not self.detected_packages: raise PythonThriftBuilder.CodeGenerationException( 'No Thrift structures declared in %s!' % self.target) def dump_setup_py(packages, namespace_packages): boilerplate = """ from setuptools import setup setup(name = "%(target_name)s", version = "dev", description = "autogenerated thrift bindings for %(target_name)s", package_dir = { "": "gen-py" }, packages = %(packages)s, namespace_packages = %(namespace_packages)s) """ boilerplate = boilerplate % { 'target_name': self.target.name, 'genpy_root': genpy_root, 'packages': repr(list(packages)), 'namespace_packages': repr(list(namespace_packages)) } self.chroot.write(boilerplate, os.path.join(self.codegen_root, 'setup.py')) dump_setup_py(self.detected_packages, self.detected_namespace_packages) egg_root = os.path.join(self.chroot.path(), self.codegen_root) egg_path = EggBuilder().build_egg(egg_root, self.target) return egg_path
class PythonAntlrBuilder(object): """ Antlr builder. """ class CodeGenerationException(Exception): pass def __init__(self, target, root_dir): self.target = target self.root = root_dir self.module = target.module distdir = os.path.join(self.root, 'dist') self.chroot = Chroot(root_dir, distdir, target.name) codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.') self.codegen_root = os.path.relpath(codegen_root, self.chroot.path()) self.created_packages = set() self.created_namespace_packages = set() def __del__(self): self.cleanup() def packages(self): return self.created_packages def cleanup(self): shutil.rmtree(self.chroot.path()) def run_antlrs(self, output_dir): args = [ 'java', '-jar', os.path.join(self.root, 'build-support/ivy/lib/ivy-2.2.0.jar'), '-settings', os.path.join(self.root, 'build-support/ivy/ivysettings.xml'), '-dependency', 'org.antlr', 'antlr', self.target.antlr_version, '-types', 'jar', '-main', 'org.antlr.Tool', '--', '-fo', output_dir ] for source in self.target.sources: abs_path = os.path.abspath( os.path.join(self.root, self.target.target_base, source)) args.append(abs_path) cwd = os.getcwd() os.chdir(self.chroot.path()) print 'PythonAntlrBuilder executing: %s' % ' '.join(args) try: po = subprocess.Popen(args) finally: os.chdir(cwd) rv = po.wait() if rv != 0: comm = po.communicate() print >> sys.stderr, 'ANTLR generation failed!' print >> sys.stderr, 'STDOUT' print >> sys.stderr, comm[0] print >> sys.stderr, 'STDERR' print >> sys.stderr, comm[1] return rv == 0 @staticmethod def path_to_module(path): return path.replace(os.path.sep, '.') def build_egg(self): gen_root = os.path.join(self.chroot.path(), self.codegen_root) # Create the package structure. path = gen_root package = '' for module_name in self.module.split('.'): path = os.path.join(path, module_name) if package == '': package = module_name else: package = package + '.' + module_name os.mkdir(path) with open(os.path.join(path, '__init__.py'), 'w') as f: if package != self.module: # Only write this in the non-leaf modules. f.write( "__import__('pkg_resources').declare_namespace(__name__)" ) self.created_namespace_packages.add(package) self.created_packages.add(package) # autogenerate the python files that we bundle up self.run_antlrs(path) def dump_setup_py(packages, namespace_packages): boilerplate = """ from setuptools import setup setup(name = "%(target_name)s", version = "dev", description = "autogenerated ANTLR parsers for %(target_name)s", package_dir = { "": "." }, packages = %(packages)s, namespace_packages = %(namespace_packages)s) """ boilerplate = boilerplate % { 'target_name': self.target.name, 'packages': repr(packages), 'namespace_packages': repr(list(namespace_packages)) } self.chroot.write(boilerplate, os.path.join(self.codegen_root, 'setup.py')) dump_setup_py(self.created_packages, self.created_namespace_packages) egg_root = os.path.join(self.chroot.path(), self.codegen_root) egg_path = EggBuilder().build_egg(egg_root, self.target) return egg_path
class PythonChroot(object): class InvalidDependencyException(Exception): def __init__(self, target): Exception.__init__( self, "Not a valid Python dependency! Found: %s" % target) class BuildFailureException(Exception): def __init__(self, target): Exception.__init__( self, "Not a valid Python dependency! Found: %s" % target) def __init__(self, target, root_dir): self.target = target self.root = root_dir distdir = os.path.join(root_dir, 'dist') self.chroot = Chroot(root_dir, distdir, target.name) def __del__(self): shutil.rmtree(self.chroot.path()) def path(self): return self.chroot.path() def _dump_library(self, library): def translate_module(module): if module is None: module = '' return module.replace('.', os.path.sep) def copy_to_chroot(base, path, relative_to=None, label=None): src = os.path.join(base, path) dst = os.path.join(translate_module(relative_to), path) self.chroot.copy(src, dst, label) template = library._create_template_data() print ' Dumping library: %s [relative module: %s]' % (library, template.module) for filename in template.sources: copy_to_chroot(template.template_base, filename, template.module, 'sources') for filename in template.resources: copy_to_chroot(template.template_base, filename, template.module, 'resources') def _dump_inits(self): # iterate through self.digest and find missing __init__.py's relative_digest = self.chroot.get('sources') init_digest = set() for path in relative_digest: split_path = path.split(os.path.sep) for k in range(1, len(split_path)): sub_path = os.path.sep.join(split_path[0:k] + ['__init__.py']) if sub_path not in relative_digest and sub_path not in init_digest: print ' Dumping __init__: %s' % sub_path self.chroot.touch(sub_path) init_digest.add(sub_path) def _dump_egg(self, egg): print ' Dumping egg: %s' % egg for egg_path in egg.eggs: src = egg_path dst = os.path.join('.deps', os.path.basename(src)) if os.path.isfile(src): self.chroot.copy(src, dst, 'resources') else: for src_dir, subdirs, files in os.walk(src): for f in files: self.chroot.copy( os.path.join(src_dir, f), os.path.join( dst, os.path.relpath(os.path.join(src_dir, f), src)), 'resources') def _dump_setuptools(self): SETUPTOOLS = 'setuptools-0.6c11-py2.6.egg' print ' Dumping setuptools: %s' % SETUPTOOLS data = pkgutil.get_data(__name__, os.path.join('bootstrap', SETUPTOOLS)) dst = os.path.join('.deps', SETUPTOOLS) self.chroot.write(data, dst, 'resources') def _dump_bin(self, binary_name, base): src = os.path.join(base, binary_name) print ' Dumping binary: %s' % binary_name self.chroot.copy(src, '__main__.py', 'sources') def _dump_thrift_library(self, library): print ' Generating %s...' % library self._dump_built_library(PythonThriftBuilder(library, self.root)) def _dump_antlr_library(self, library): print ' Generating %s...' % library self._dump_built_library(PythonAntlrBuilder(library, self.root)) def _dump_built_library(self, builder): egg_file = builder.build_egg() if egg_file: egg_file = os.path.relpath(egg_file, self.root) for pkg in builder.packages(): print ' found namespace: %s' % pkg # make a random string to disambiguate possibly similarly-named eggs? randstr = ''.join( map(chr, random.sample(range(ord('a'), ord('z')), 8))) + '_' print ' copying...', self.chroot.copy( egg_file, os.path.join('.deps', randstr + os.path.basename(egg_file)), 'resources') print 'done.' else: print ' Failed!' raise PythonChroot.BuildFailureException("Failed to build %s!" % library) def build_dep_tree(self, target): libraries = set() eggs = set() binaries = set() thrifts = set() antlrs = set() def add_dep(trg): if isinstance(trg, PythonLibrary): if trg.sources: libraries.add(trg) for egg in [ dep for dep in trg.dependencies if isinstance(dep, PythonEgg) ]: eggs.add(egg) elif isinstance(trg, PythonEgg): eggs.add(trg) elif isinstance(trg, PythonBinary): binaries.add(trg) elif isinstance(trg, PythonThriftLibrary): thrifts.add(trg) elif isinstance(trg, PythonAntlrLibrary): antlrs.add(trg) elif isinstance(trg, PythonTests): # do not dump test sources/resources, but dump their # dependencies. pass else: raise PythonChroot.InvalidDependencyException(trg) return [ dep for dep in trg.dependencies if not isinstance(dep, PythonEgg) ] target.walk(lambda t: add_dep(t), lambda typ: not isinstance(typ, PythonEgg)) return libraries, eggs, binaries, thrifts, antlrs def dump(self): print 'Building PythonBinary %s:' % self.target libraries, eggs, binaries, thrifts, antlrs = self.build_dep_tree( self.target) for lib in libraries: self._dump_library(lib) self._dump_inits() if eggs: self._dump_setuptools() for egg in eggs: self._dump_egg(egg) for thr in thrifts: self._dump_thrift_library(thr) for antlr in antlrs: self._dump_antlr_library(antlr) if len(binaries) > 1: print >> sys.stderr, 'WARNING: Target has multiple python_binary targets!' for binary in binaries: self._dump_bin(binary.sources[0], binary.target_base) return self.chroot
class PythonThriftBuilder(object): """ Thrift builder. """ class UnknownPlatformException(Exception): def __init__(self, platform): Exception.__init__(self, "Unknown platform: %s!" % platform) class CodeGenerationException(Exception): pass def __init__(self, target, root_dir): self.target = target self.root = root_dir distdir = os.path.join(self.root, 'dist') self.chroot = Chroot(root_dir, distdir, target.name) codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.') self.codegen_root = os.path.relpath(codegen_root, self.chroot.path()) self.detected_packages = set() def __del__(self): self.cleanup() def packages(self): return self.detected_packages def cleanup(self): shutil.rmtree(self.chroot.path()) @staticmethod def thrift_binary(root_dir): # TODO(Brian Wickman): Will we ever need to make this smarter? PLATMAP = { 'darwin': ['mac', '10.6', '0.5.0', 'thrift'], 'linux': ['linux', 'x86_64', '0.5.0', 'thrift'], } platform = os.uname()[0].lower() if platform not in PLATMAP: raise PythonThriftBuilder.UnknownPlatformException(platform) return os.path.join(root_dir, 'build-support', 'bin', 'thrift', *PLATMAP[platform]) def run_thrifts(self): for src in self.target.sources: if not self._run_thrift(src): raise PythonThriftBuilder.CodeGenerationException( "Could not generate .py from %s!" % src) def _run_thrift(self, source): thrift_file = source thrift_abs_path = os.path.join(self.root, self.target.target_base, thrift_file) thrift_abs_path = os.path.abspath(thrift_abs_path) # pushd, call, popd cwd = os.getcwd() os.chdir(self.chroot.path()) po = subprocess.Popen([ PythonThriftBuilder.thrift_binary(self.root), '--gen', 'py:new_style', '-o', self.codegen_root, thrift_abs_path]) os.chdir(cwd) rv = po.wait() if rv != 0: comm = po.communicate() print >> sys.stderr, 'thrift generation failed!' print >> sys.stderr, 'STDOUT' print >> sys.stderr, comm[0] print >> sys.stderr, 'STDERR' print >> sys.stderr, comm[1] return rv == 0 @staticmethod def path_to_module(path): return path.replace(os.path.sep, '.') def build_egg(self): # autogenerate the python files that we bundle up self.run_thrifts() genpy_root = os.path.join(self.chroot.path(), self.codegen_root, 'gen-py') for dir, subdirs, files in os.walk(os.path.normpath(genpy_root)): reldir = os.path.relpath(dir, genpy_root) if reldir == '.': continue if '__init__.py' not in files: continue self.detected_packages.add(self.path_to_module(reldir)) if not self.detected_packages: raise PythonThriftBuilder.CodeGenerationException( 'No Thrift structures declared in %s!' % self.target) def dump_setup_py(packages): boilerplate = """ from setuptools import setup setup(name = "%(target_name)s", version = "dev", description = "autogenerated thrift bindings for %(target_name)s", package_dir = { "": "gen-py" }, packages = %(packages)s) """ boilerplate = boilerplate % { 'target_name': self.target.name, 'genpy_root': genpy_root, 'packages': repr(list(packages)) } self.chroot.write(boilerplate, os.path.join(self.codegen_root, 'setup.py')) dump_setup_py(self.detected_packages) egg_path = None # TODO(Brian Wickman): Do a sanity check somewhere to ensure that # setuptools is on the path? cwd = os.getcwd() egg_root = os.path.join(self.chroot.path(), self.codegen_root) os.chdir(egg_root) po = subprocess.Popen([ sys.executable, 'setup.py', 'bdist_egg', '--dist-dir=dist', '--bdist-dir=build.%s' % self.target.name], stderr=subprocess.PIPE, stdout=subprocess.PIPE) rv = po.wait() if rv == 0: eggs = os.path.abspath(os.path.join('dist', '*.egg')) eggs = glob.glob(eggs) if len(eggs) != 1: comm = po.communicate() print >> sys.stderr, 'egg generation failed' print >> sys.stderr, 'STDOUT' print >> sys.stderr, comm[0] print >> sys.stderr, 'STDERR' print >> sys.stderr, comm[1] raise PythonThriftBuilder.CodeGenerationException( 'Generation of Thrift eggs failed for target = %s' % self.target) egg_path = eggs[0] os.chdir(cwd) return egg_path
class PythonThriftBuilder(object): """ Thrift builder. """ class UnknownPlatformException(Exception): def __init__(self, platform): Exception.__init__(self, "Unknown platform: %s!" % str(platform)) class CodeGenerationException(Exception): pass def __init__(self, target, root_dir): self.target = target self.root = root_dir distdir = os.path.join(self.root, 'dist') self.chroot = Chroot(root_dir, distdir, target.name) codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.') self.codegen_root = os.path.relpath(codegen_root, self.chroot.path()) self.detected_packages = set() self.detected_namespace_packages = set() def __del__(self): self.cleanup() def packages(self): return self.detected_packages def cleanup(self): shutil.rmtree(self.chroot.path()) @staticmethod def thrift_binary(root_dir): # TODO(Brian Wickman): Will we ever need to make this smarter? PLATMAP = { ('darwin', 'i386'): ['mac', '10.6', '0.5.0', 'thrift'], ('darwin', 'x86_64'): ['mac', '10.6', '0.5.0', 'thrift'], ('darwin', 'x86_64'): ['mac', '10.7', '0.5.0', 'thrift'], ('linux', 'x86_64'): ['linux', 'x86_64', '0.5.0', 'thrift'], ('linux', 'i686') : ['linux', 'i386', '0.5.0', 'thrift'] } uname = os.uname() platform = (uname[0].lower(), uname[4]) if platform not in PLATMAP: raise PythonThriftBuilder.UnknownPlatformException(platform) return os.path.join(root_dir, 'build-support', 'bin', 'thrift', *PLATMAP[platform]) def run_thrifts(self): for src in self.target.sources: if not self._run_thrift(src): raise PythonThriftBuilder.CodeGenerationException( "Could not generate .py from %s!" % src) def _run_thrift(self, source): thrift_file = source thrift_abs_path = os.path.join(self.root, self.target.target_base, thrift_file) thrift_abs_path = os.path.abspath(thrift_abs_path) args = [ PythonThriftBuilder.thrift_binary(self.root), '--gen', 'py:new_style', '-o', self.codegen_root, thrift_abs_path] cwd = os.getcwd() os.chdir(self.chroot.path()) print 'PythonThriftBuilder executing: %s' % ' '.join(args) try: po = subprocess.Popen(args) finally: os.chdir(cwd) rv = po.wait() if rv != 0: comm = po.communicate() print >> sys.stderr, 'thrift generation failed!' print >> sys.stderr, 'STDOUT' print >> sys.stderr, comm[0] print >> sys.stderr, 'STDERR' print >> sys.stderr, comm[1] return rv == 0 @staticmethod def path_to_module(path): return path.replace(os.path.sep, '.') def build_egg(self): # autogenerate the python files that we bundle up self.run_thrifts() genpy_root = os.path.join(self.chroot.path(), self.codegen_root, 'gen-py') for dir, subdirs, files in os.walk(os.path.normpath(genpy_root)): reldir = os.path.relpath(dir, genpy_root) if reldir == '.': continue if '__init__.py' not in files: continue init_py_abspath = os.path.join(dir, '__init__.py') module_path = self.path_to_module(reldir) self.detected_packages.add(module_path) # A namespace package is one that is just a container for other # modules and subpackages. Setting their __init__.py files as follows # allows them to be distributed across multiple eggs. Without this you # couldn't have this egg share any package prefix with any other module # in any other egg or in the source tree. # # Note that the thrift compiler should always generate empty __init__.py # files, but we test for this anyway, just in case that changes. if len(files) == 1 and os.path.getsize(init_py_abspath) == 0: with open(init_py_abspath, 'w') as f: f.write("__import__('pkg_resources').declare_namespace(__name__)") self.detected_namespace_packages.add(module_path) if not self.detected_packages: raise PythonThriftBuilder.CodeGenerationException( 'No Thrift structures declared in %s!' % self.target) def dump_setup_py(packages, namespace_packages): boilerplate = """ from setuptools import setup setup(name = "%(target_name)s", version = "dev", description = "autogenerated thrift bindings for %(target_name)s", package_dir = { "": "gen-py" }, packages = %(packages)s, namespace_packages = %(namespace_packages)s) """ boilerplate = boilerplate % { 'target_name': self.target.name, 'genpy_root': genpy_root, 'packages': repr(list(packages)), 'namespace_packages': repr(list(namespace_packages)) } self.chroot.write(boilerplate, os.path.join(self.codegen_root, 'setup.py')) dump_setup_py(self.detected_packages, self.detected_namespace_packages) egg_root = os.path.join(self.chroot.path(), self.codegen_root) egg_path = EggBuilder().build_egg(egg_root, self.target) return egg_path