def __init__(self, target, root_dir):
self.target = target
self.root = root_dir
distdir = os.path.join(self.root, 'dist')
self.chroot = Chroot(root_dir, distdir, target.name)
codegen_root = tempfile.mkdtemp(dir=self.chroot.path(),
prefix='codegen.')
self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
self.detected_packages = set()
self.detected_namespace_packages = set()
def __init__(self, target, root_dir): self.target = target self.root = root_dir distdir = os.path.join(self.root, 'dist') self.chroot = Chroot(root_dir, distdir, target.name) codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.') self.codegen_root = os.path.relpath(codegen_root, self.chroot.path()) self.detected_packages = set()
def __init__(self, target, root_dir): self.target = target self.root = root_dir distdir = os.path.join(root_dir, 'dist') self.chroot = Chroot(root_dir, distdir, target.name)
class PythonChroot(object):
class InvalidDependencyException(Exception):
def __init__(self, target):
Exception.__init__(self, "Not a valid Python dependency! Found: %s" % target)
class BuildFailureException(Exception):
def __init__(self, target):
Exception.__init__(self, "Not a valid Python dependency! Found: %s" % target)
def __init__(self, target, root_dir):
self.target = target
self.root = root_dir
distdir = os.path.join(root_dir, 'dist')
self.chroot = Chroot(root_dir, distdir, target.name)
def __del__(self):
shutil.rmtree(self.chroot.path())
def path(self):
return self.chroot.path()
def _dump_library(self, library):
def translate_module(module):
if module is None:
module=''
return module.replace('.', os.path.sep)
def copy_to_chroot(base, path, relative_to=None, label=None):
src = os.path.join(base, path)
dst = os.path.join(translate_module(relative_to), path)
self.chroot.copy(src, dst, label)
template = library._create_template_data()
print ' Dumping library: %s [relative module: %s]' % (library, template.module)
for filename in template.sources:
copy_to_chroot(template.template_base, filename, template.module, 'sources')
for filename in template.resources:
copy_to_chroot(template.template_base, filename, template.module, 'resources')
def _dump_inits(self):
# iterate through self.digest and find missing __init__.py's
relative_digest = self.chroot.get('sources')
init_digest = set()
for path in relative_digest:
split_path = path.split(os.path.sep)
for k in range(1, len(split_path)):
sub_path = os.path.sep.join(split_path[0:k] + ['__init__.py'])
if sub_path not in relative_digest and sub_path not in init_digest:
print ' Dumping __init__: %s' % sub_path
self.chroot.touch(sub_path)
init_digest.add(sub_path)
def _dump_egg(self, egg):
print ' Dumping egg: %s' % egg
for egg_path in egg.eggs:
src = egg_path
dst = os.path.join('.deps', os.path.basename(src))
self.chroot.copy(src, dst, 'resources')
def _dump_setuptools(self):
SETUPTOOLS = 'setuptools-0.6c11-py2.6.egg'
print ' Dumping setuptools: %s' % SETUPTOOLS
data = pkgutil.get_data(__name__, os.path.join('bootstrap', SETUPTOOLS))
dst = os.path.join('.deps', SETUPTOOLS)
self.chroot.write(data, dst, 'resources')
def _dump_bin(self, binary_name, base):
src = os.path.join(base, binary_name)
print ' Dumping binary: %s' % binary_name
self.chroot.copy(src, '__main__.py', 'sources')
def _dump_thrift_library(self, library):
builder = PythonThriftBuilder(library, self.root)
print ' Generating %s...' % library
egg_file = builder.build_egg()
if egg_file:
egg_file = os.path.relpath(egg_file, self.root)
for pkg in builder.packages():
print ' found namespace: %s' % pkg
# make a random string to disambiguate possibly similarly-named eggs?
randstr = ''.join(map(chr, random.sample(range(ord('a'), ord('z')), 8))) + '_'
print ' copying...',
self.chroot.copy(egg_file, os.path.join('.deps', randstr + os.path.basename(egg_file)), 'resources')
print 'done.'
else:
print ' Failed!'
raise PythonChroot.BuildFailureException(
"Failed to build %s!" % library)
def build_dep_tree(self, target):
libraries = set()
eggs = set()
binaries = set()
thrifts = set()
def add_dep(trg):
if isinstance(trg, PythonLibrary):
if trg.sources:
libraries.add(trg)
for egg in [dep for dep in trg.dependencies if isinstance(dep, PythonEgg)]:
eggs.add(egg)
elif isinstance(trg, PythonEgg):
eggs.add(trg)
elif isinstance(trg, PythonBinary):
binaries.add(trg)
elif isinstance(trg, PythonThriftLibrary):
thrifts.add(trg)
elif isinstance(trg, PythonTests):
# do not dump test sources/resources, but dump their
# dependencies.
pass
else:
raise PythonChroot.InvalidDependencyException(trg)
return [dep for dep in trg.dependencies if not isinstance(dep, PythonEgg)]
target.walk(lambda t: add_dep(t), lambda typ: not isinstance(typ, PythonEgg))
return libraries, eggs, binaries, thrifts
def dump(self):
print 'Building PythonBinary %s:' % self.target
libraries, eggs, binaries, thrifts = self.build_dep_tree(self.target)
for lib in libraries:
self._dump_library(lib)
self._dump_inits()
if eggs:
self._dump_setuptools()
for egg in eggs:
self._dump_egg(egg)
for thr in thrifts:
self._dump_thrift_library(thr)
if len(binaries) > 1:
print >> sys.stderr, 'WARNING: Target has multiple python_binary targets!'
for binary in binaries:
self._dump_bin(binary.sources[0], binary.target_base)
return self.chroot
class PythonAntlrBuilder(object):
"""
Antlr builder.
"""
class CodeGenerationException(Exception): pass
def __init__(self, target, root_dir):
self.target = target
self.root = root_dir
self.module = target.module
distdir = os.path.join(self.root, 'dist')
self.chroot = Chroot(root_dir, distdir, target.name)
codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.')
self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
self.created_packages = set()
self.created_namespace_packages = set()
def __del__(self):
self.cleanup()
def packages(self):
return self.created_packages
def cleanup(self):
shutil.rmtree(self.chroot.path())
def run_antlrs(self, output_dir):
args = [
'java',
'-jar',
os.path.join(self.root, 'build-support/ivy/lib/ivy-2.2.0.jar'),
'-settings',
os.path.join(self.root, 'build-support/ivy/ivysettings.xml'),
'-dependency',
'org.antlr',
'antlr',
self.target.antlr_version,
'-types',
'jar',
'-main',
'org.antlr.Tool',
'--',
'-fo',
output_dir]
for source in self.target.sources:
abs_path = os.path.abspath(os.path.join(self.root, self.target.target_base, source))
args.append(abs_path)
cwd = os.getcwd()
os.chdir(self.chroot.path())
print 'PythonAntlrBuilder executing: %s' % ' '.join(args)
try:
po = subprocess.Popen(args)
finally:
os.chdir(cwd)
rv = po.wait()
if rv != 0:
comm = po.communicate()
print >> sys.stderr, 'ANTLR generation failed!'
print >> sys.stderr, 'STDOUT'
print >> sys.stderr, comm[0]
print >> sys.stderr, 'STDERR'
print >> sys.stderr, comm[1]
return rv == 0
@staticmethod
def path_to_module(path):
return path.replace(os.path.sep, '.')
def build_egg(self):
gen_root = os.path.join(self.chroot.path(), self.codegen_root)
# Create the package structure.
path = gen_root
package = ''
for module_name in self.module.split('.'):
path = os.path.join(path, module_name)
if package == '':
package = module_name
else:
package = package + '.' + module_name
os.mkdir(path)
with open(os.path.join(path, '__init__.py'), 'w') as f:
if package != self.module: # Only write this in the non-leaf modules.
f.write("__import__('pkg_resources').declare_namespace(__name__)")
self.created_namespace_packages.add(package)
self.created_packages.add(package)
# autogenerate the python files that we bundle up
self.run_antlrs(path)
def dump_setup_py(packages, namespace_packages):
boilerplate = """
from setuptools import setup
setup(name = "%(target_name)s",
version = "dev",
description = "autogenerated ANTLR parsers for %(target_name)s",
package_dir = { "": "." },
packages = %(packages)s,
namespace_packages = %(namespace_packages)s)
"""
boilerplate = boilerplate % {
'target_name': self.target.name,
'packages': repr(packages),
'namespace_packages': repr(list(namespace_packages))
}
self.chroot.write(boilerplate, os.path.join(self.codegen_root, 'setup.py'))
dump_setup_py(self.created_packages, self.created_namespace_packages)
egg_root = os.path.join(self.chroot.path(), self.codegen_root)
egg_path = EggBuilder().build_egg(egg_root, self.target)
return egg_path
class PythonThriftBuilder(object):
"""
Thrift builder.
"""
class UnknownPlatformException(Exception):
def __init__(self, platform):
Exception.__init__(self, "Unknown platform: %s!" % str(platform))
class CodeGenerationException(Exception):
pass
def __init__(self, target, root_dir):
self.target = target
self.root = root_dir
distdir = os.path.join(self.root, 'dist')
self.chroot = Chroot(root_dir, distdir, target.name)
codegen_root = tempfile.mkdtemp(dir=self.chroot.path(),
prefix='codegen.')
self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
self.detected_packages = set()
self.detected_namespace_packages = set()
def __del__(self):
self.cleanup()
def packages(self):
return self.detected_packages
def cleanup(self):
shutil.rmtree(self.chroot.path())
@staticmethod
def thrift_binary(root_dir):
# TODO(Brian Wickman): Will we ever need to make this smarter?
PLATMAP = {
('darwin', 'i386'): ['mac', '10.6', '0.5.0', 'thrift'],
('darwin', 'x86_64'): ['mac', '10.6', '0.5.0', 'thrift'],
('darwin', 'x86_64'): ['mac', '10.7', '0.5.0', 'thrift'],
('linux', 'x86_64'): ['linux', 'x86_64', '0.5.0', 'thrift'],
('linux', 'i686'): ['linux', 'i386', '0.5.0', 'thrift']
}
uname = os.uname()
platform = (uname[0].lower(), uname[4])
if platform not in PLATMAP:
raise PythonThriftBuilder.UnknownPlatformException(platform)
return os.path.join(root_dir, 'build-support', 'bin', 'thrift',
*PLATMAP[platform])
def run_thrifts(self):
for src in self.target.sources:
if not self._run_thrift(src):
raise PythonThriftBuilder.CodeGenerationException(
"Could not generate .py from %s!" % src)
def _run_thrift(self, source):
thrift_file = source
thrift_abs_path = os.path.join(self.root, self.target.target_base,
thrift_file)
thrift_abs_path = os.path.abspath(thrift_abs_path)
args = [
PythonThriftBuilder.thrift_binary(self.root), '--gen',
'py:new_style', '-o', self.codegen_root, thrift_abs_path
]
cwd = os.getcwd()
os.chdir(self.chroot.path())
print 'PythonThriftBuilder executing: %s' % ' '.join(args)
try:
po = subprocess.Popen(args)
finally:
os.chdir(cwd)
rv = po.wait()
if rv != 0:
comm = po.communicate()
print >> sys.stderr, 'thrift generation failed!'
print >> sys.stderr, 'STDOUT'
print >> sys.stderr, comm[0]
print >> sys.stderr, 'STDERR'
print >> sys.stderr, comm[1]
return rv == 0
@staticmethod
def path_to_module(path):
return path.replace(os.path.sep, '.')
def build_egg(self):
# autogenerate the python files that we bundle up
self.run_thrifts()
genpy_root = os.path.join(self.chroot.path(), self.codegen_root,
'gen-py')
for dir, subdirs, files in os.walk(os.path.normpath(genpy_root)):
reldir = os.path.relpath(dir, genpy_root)
if reldir == '.': continue
if '__init__.py' not in files: continue
init_py_abspath = os.path.join(dir, '__init__.py')
module_path = self.path_to_module(reldir)
self.detected_packages.add(module_path)
# A namespace package is one that is just a container for other
# modules and subpackages. Setting their __init__.py files as follows
# allows them to be distributed across multiple eggs. Without this you
# couldn't have this egg share any package prefix with any other module
# in any other egg or in the source tree.
#
# Note that the thrift compiler should always generate empty __init__.py
# files, but we test for this anyway, just in case that changes.
if len(files) == 1 and os.path.getsize(init_py_abspath) == 0:
with open(init_py_abspath, 'w') as f:
f.write(
"__import__('pkg_resources').declare_namespace(__name__)"
)
self.detected_namespace_packages.add(module_path)
if not self.detected_packages:
raise PythonThriftBuilder.CodeGenerationException(
'No Thrift structures declared in %s!' % self.target)
def dump_setup_py(packages, namespace_packages):
boilerplate = """
from setuptools import setup
setup(name = "%(target_name)s",
version = "dev",
description = "autogenerated thrift bindings for %(target_name)s",
package_dir = { "": "gen-py" },
packages = %(packages)s,
namespace_packages = %(namespace_packages)s)
"""
boilerplate = boilerplate % {
'target_name': self.target.name,
'genpy_root': genpy_root,
'packages': repr(list(packages)),
'namespace_packages': repr(list(namespace_packages))
}
self.chroot.write(boilerplate,
os.path.join(self.codegen_root, 'setup.py'))
dump_setup_py(self.detected_packages, self.detected_namespace_packages)
egg_root = os.path.join(self.chroot.path(), self.codegen_root)
egg_path = EggBuilder().build_egg(egg_root, self.target)
return egg_path
class PythonAntlrBuilder(object):
"""
Antlr builder.
"""
class CodeGenerationException(Exception):
pass
def __init__(self, target, root_dir):
self.target = target
self.root = root_dir
self.module = target.module
distdir = os.path.join(self.root, 'dist')
self.chroot = Chroot(root_dir, distdir, target.name)
codegen_root = tempfile.mkdtemp(dir=self.chroot.path(),
prefix='codegen.')
self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
self.created_packages = set()
self.created_namespace_packages = set()
def __del__(self):
self.cleanup()
def packages(self):
return self.created_packages
def cleanup(self):
shutil.rmtree(self.chroot.path())
def run_antlrs(self, output_dir):
args = [
'java', '-jar',
os.path.join(self.root,
'build-support/ivy/lib/ivy-2.2.0.jar'), '-settings',
os.path.join(self.root, 'build-support/ivy/ivysettings.xml'),
'-dependency', 'org.antlr', 'antlr', self.target.antlr_version,
'-types', 'jar', '-main', 'org.antlr.Tool', '--', '-fo', output_dir
]
for source in self.target.sources:
abs_path = os.path.abspath(
os.path.join(self.root, self.target.target_base, source))
args.append(abs_path)
cwd = os.getcwd()
os.chdir(self.chroot.path())
print 'PythonAntlrBuilder executing: %s' % ' '.join(args)
try:
po = subprocess.Popen(args)
finally:
os.chdir(cwd)
rv = po.wait()
if rv != 0:
comm = po.communicate()
print >> sys.stderr, 'ANTLR generation failed!'
print >> sys.stderr, 'STDOUT'
print >> sys.stderr, comm[0]
print >> sys.stderr, 'STDERR'
print >> sys.stderr, comm[1]
return rv == 0
@staticmethod
def path_to_module(path):
return path.replace(os.path.sep, '.')
def build_egg(self):
gen_root = os.path.join(self.chroot.path(), self.codegen_root)
# Create the package structure.
path = gen_root
package = ''
for module_name in self.module.split('.'):
path = os.path.join(path, module_name)
if package == '':
package = module_name
else:
package = package + '.' + module_name
os.mkdir(path)
with open(os.path.join(path, '__init__.py'), 'w') as f:
if package != self.module: # Only write this in the non-leaf modules.
f.write(
"__import__('pkg_resources').declare_namespace(__name__)"
)
self.created_namespace_packages.add(package)
self.created_packages.add(package)
# autogenerate the python files that we bundle up
self.run_antlrs(path)
def dump_setup_py(packages, namespace_packages):
boilerplate = """
from setuptools import setup
setup(name = "%(target_name)s",
version = "dev",
description = "autogenerated ANTLR parsers for %(target_name)s",
package_dir = { "": "." },
packages = %(packages)s,
namespace_packages = %(namespace_packages)s)
"""
boilerplate = boilerplate % {
'target_name': self.target.name,
'packages': repr(packages),
'namespace_packages': repr(list(namespace_packages))
}
self.chroot.write(boilerplate,
os.path.join(self.codegen_root, 'setup.py'))
dump_setup_py(self.created_packages, self.created_namespace_packages)
egg_root = os.path.join(self.chroot.path(), self.codegen_root)
egg_path = EggBuilder().build_egg(egg_root, self.target)
return egg_path
def __init__(self, target, root_dir):
self.target = target
self.root = root_dir
distdir = os.path.join(root_dir, 'dist')
self.chroot = Chroot(root_dir, distdir, target.name)
class PythonChroot(object):
class InvalidDependencyException(Exception):
def __init__(self, target):
Exception.__init__(
self, "Not a valid Python dependency! Found: %s" % target)
class BuildFailureException(Exception):
def __init__(self, target):
Exception.__init__(
self, "Not a valid Python dependency! Found: %s" % target)
def __init__(self, target, root_dir):
self.target = target
self.root = root_dir
distdir = os.path.join(root_dir, 'dist')
self.chroot = Chroot(root_dir, distdir, target.name)
def __del__(self):
shutil.rmtree(self.chroot.path())
def path(self):
return self.chroot.path()
def _dump_library(self, library):
def translate_module(module):
if module is None:
module = ''
return module.replace('.', os.path.sep)
def copy_to_chroot(base, path, relative_to=None, label=None):
src = os.path.join(base, path)
dst = os.path.join(translate_module(relative_to), path)
self.chroot.copy(src, dst, label)
template = library._create_template_data()
print ' Dumping library: %s [relative module: %s]' % (library,
template.module)
for filename in template.sources:
copy_to_chroot(template.template_base, filename, template.module,
'sources')
for filename in template.resources:
copy_to_chroot(template.template_base, filename, template.module,
'resources')
def _dump_inits(self):
# iterate through self.digest and find missing __init__.py's
relative_digest = self.chroot.get('sources')
init_digest = set()
for path in relative_digest:
split_path = path.split(os.path.sep)
for k in range(1, len(split_path)):
sub_path = os.path.sep.join(split_path[0:k] + ['__init__.py'])
if sub_path not in relative_digest and sub_path not in init_digest:
print ' Dumping __init__: %s' % sub_path
self.chroot.touch(sub_path)
init_digest.add(sub_path)
def _dump_egg(self, egg):
print ' Dumping egg: %s' % egg
for egg_path in egg.eggs:
src = egg_path
dst = os.path.join('.deps', os.path.basename(src))
if os.path.isfile(src):
self.chroot.copy(src, dst, 'resources')
else:
for src_dir, subdirs, files in os.walk(src):
for f in files:
self.chroot.copy(
os.path.join(src_dir, f),
os.path.join(
dst,
os.path.relpath(os.path.join(src_dir, f),
src)), 'resources')
def _dump_setuptools(self):
SETUPTOOLS = 'setuptools-0.6c11-py2.6.egg'
print ' Dumping setuptools: %s' % SETUPTOOLS
data = pkgutil.get_data(__name__, os.path.join('bootstrap',
SETUPTOOLS))
dst = os.path.join('.deps', SETUPTOOLS)
self.chroot.write(data, dst, 'resources')
def _dump_bin(self, binary_name, base):
src = os.path.join(base, binary_name)
print ' Dumping binary: %s' % binary_name
self.chroot.copy(src, '__main__.py', 'sources')
def _dump_thrift_library(self, library):
print ' Generating %s...' % library
self._dump_built_library(PythonThriftBuilder(library, self.root))
def _dump_antlr_library(self, library):
print ' Generating %s...' % library
self._dump_built_library(PythonAntlrBuilder(library, self.root))
def _dump_built_library(self, builder):
egg_file = builder.build_egg()
if egg_file:
egg_file = os.path.relpath(egg_file, self.root)
for pkg in builder.packages():
print ' found namespace: %s' % pkg
# make a random string to disambiguate possibly similarly-named eggs?
randstr = ''.join(
map(chr, random.sample(range(ord('a'), ord('z')), 8))) + '_'
print ' copying...',
self.chroot.copy(
egg_file,
os.path.join('.deps', randstr + os.path.basename(egg_file)),
'resources')
print 'done.'
else:
print ' Failed!'
raise PythonChroot.BuildFailureException("Failed to build %s!" %
library)
def build_dep_tree(self, target):
libraries = set()
eggs = set()
binaries = set()
thrifts = set()
antlrs = set()
def add_dep(trg):
if isinstance(trg, PythonLibrary):
if trg.sources:
libraries.add(trg)
for egg in [
dep for dep in trg.dependencies
if isinstance(dep, PythonEgg)
]:
eggs.add(egg)
elif isinstance(trg, PythonEgg):
eggs.add(trg)
elif isinstance(trg, PythonBinary):
binaries.add(trg)
elif isinstance(trg, PythonThriftLibrary):
thrifts.add(trg)
elif isinstance(trg, PythonAntlrLibrary):
antlrs.add(trg)
elif isinstance(trg, PythonTests):
# do not dump test sources/resources, but dump their
# dependencies.
pass
else:
raise PythonChroot.InvalidDependencyException(trg)
return [
dep for dep in trg.dependencies
if not isinstance(dep, PythonEgg)
]
target.walk(lambda t: add_dep(t),
lambda typ: not isinstance(typ, PythonEgg))
return libraries, eggs, binaries, thrifts, antlrs
def dump(self):
print 'Building PythonBinary %s:' % self.target
libraries, eggs, binaries, thrifts, antlrs = self.build_dep_tree(
self.target)
for lib in libraries:
self._dump_library(lib)
self._dump_inits()
if eggs:
self._dump_setuptools()
for egg in eggs:
self._dump_egg(egg)
for thr in thrifts:
self._dump_thrift_library(thr)
for antlr in antlrs:
self._dump_antlr_library(antlr)
if len(binaries) > 1:
print >> sys.stderr, 'WARNING: Target has multiple python_binary targets!'
for binary in binaries:
self._dump_bin(binary.sources[0], binary.target_base)
return self.chroot
class PythonThriftBuilder(object):
"""
Thrift builder.
"""
class UnknownPlatformException(Exception):
def __init__(self, platform):
Exception.__init__(self, "Unknown platform: %s!" % platform)
class CodeGenerationException(Exception): pass
def __init__(self, target, root_dir):
self.target = target
self.root = root_dir
distdir = os.path.join(self.root, 'dist')
self.chroot = Chroot(root_dir, distdir, target.name)
codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.')
self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
self.detected_packages = set()
def __del__(self):
self.cleanup()
def packages(self):
return self.detected_packages
def cleanup(self):
shutil.rmtree(self.chroot.path())
@staticmethod
def thrift_binary(root_dir):
# TODO(Brian Wickman): Will we ever need to make this smarter?
PLATMAP = {
'darwin': ['mac', '10.6', '0.5.0', 'thrift'],
'linux': ['linux', 'x86_64', '0.5.0', 'thrift'],
}
platform = os.uname()[0].lower()
if platform not in PLATMAP:
raise PythonThriftBuilder.UnknownPlatformException(platform)
return os.path.join(root_dir, 'build-support', 'bin', 'thrift', *PLATMAP[platform])
def run_thrifts(self):
for src in self.target.sources:
if not self._run_thrift(src):
raise PythonThriftBuilder.CodeGenerationException(
"Could not generate .py from %s!" % src)
def _run_thrift(self, source):
thrift_file = source
thrift_abs_path = os.path.join(self.root, self.target.target_base, thrift_file)
thrift_abs_path = os.path.abspath(thrift_abs_path)
# pushd, call, popd
cwd = os.getcwd()
os.chdir(self.chroot.path())
po = subprocess.Popen([
PythonThriftBuilder.thrift_binary(self.root),
'--gen', 'py:new_style',
'-o', self.codegen_root,
thrift_abs_path])
os.chdir(cwd)
rv = po.wait()
if rv != 0:
comm = po.communicate()
print >> sys.stderr, 'thrift generation failed!'
print >> sys.stderr, 'STDOUT'
print >> sys.stderr, comm[0]
print >> sys.stderr, 'STDERR'
print >> sys.stderr, comm[1]
return rv == 0
@staticmethod
def path_to_module(path):
return path.replace(os.path.sep, '.')
def build_egg(self):
# autogenerate the python files that we bundle up
self.run_thrifts()
genpy_root = os.path.join(self.chroot.path(), self.codegen_root, 'gen-py')
for dir, subdirs, files in os.walk(os.path.normpath(genpy_root)):
reldir = os.path.relpath(dir, genpy_root)
if reldir == '.': continue
if '__init__.py' not in files: continue
self.detected_packages.add(self.path_to_module(reldir))
if not self.detected_packages:
raise PythonThriftBuilder.CodeGenerationException(
'No Thrift structures declared in %s!' % self.target)
def dump_setup_py(packages):
boilerplate = """
from setuptools import setup
setup(name = "%(target_name)s",
version = "dev",
description = "autogenerated thrift bindings for %(target_name)s",
package_dir = { "": "gen-py" },
packages = %(packages)s)
"""
boilerplate = boilerplate % {
'target_name': self.target.name,
'genpy_root': genpy_root,
'packages': repr(list(packages))
}
self.chroot.write(boilerplate, os.path.join(self.codegen_root, 'setup.py'))
dump_setup_py(self.detected_packages)
egg_path = None
# TODO(Brian Wickman): Do a sanity check somewhere to ensure that
# setuptools is on the path?
cwd = os.getcwd()
egg_root = os.path.join(self.chroot.path(), self.codegen_root)
os.chdir(egg_root)
po = subprocess.Popen([
sys.executable,
'setup.py', 'bdist_egg',
'--dist-dir=dist',
'--bdist-dir=build.%s' % self.target.name], stderr=subprocess.PIPE, stdout=subprocess.PIPE)
rv = po.wait()
if rv == 0:
eggs = os.path.abspath(os.path.join('dist', '*.egg'))
eggs = glob.glob(eggs)
if len(eggs) != 1:
comm = po.communicate()
print >> sys.stderr, 'egg generation failed'
print >> sys.stderr, 'STDOUT'
print >> sys.stderr, comm[0]
print >> sys.stderr, 'STDERR'
print >> sys.stderr, comm[1]
raise PythonThriftBuilder.CodeGenerationException(
'Generation of Thrift eggs failed for target = %s' % self.target)
egg_path = eggs[0]
os.chdir(cwd)
return egg_path
class PythonThriftBuilder(object):
"""
Thrift builder.
"""
class UnknownPlatformException(Exception):
def __init__(self, platform):
Exception.__init__(self, "Unknown platform: %s!" % str(platform))
class CodeGenerationException(Exception): pass
def __init__(self, target, root_dir):
self.target = target
self.root = root_dir
distdir = os.path.join(self.root, 'dist')
self.chroot = Chroot(root_dir, distdir, target.name)
codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.')
self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
self.detected_packages = set()
self.detected_namespace_packages = set()
def __del__(self):
self.cleanup()
def packages(self):
return self.detected_packages
def cleanup(self):
shutil.rmtree(self.chroot.path())
@staticmethod
def thrift_binary(root_dir):
# TODO(Brian Wickman): Will we ever need to make this smarter?
PLATMAP = {
('darwin', 'i386'): ['mac', '10.6', '0.5.0', 'thrift'],
('darwin', 'x86_64'): ['mac', '10.6', '0.5.0', 'thrift'],
('darwin', 'x86_64'): ['mac', '10.7', '0.5.0', 'thrift'],
('linux', 'x86_64'): ['linux', 'x86_64', '0.5.0', 'thrift'],
('linux', 'i686') : ['linux', 'i386', '0.5.0', 'thrift']
}
uname = os.uname()
platform = (uname[0].lower(), uname[4])
if platform not in PLATMAP:
raise PythonThriftBuilder.UnknownPlatformException(platform)
return os.path.join(root_dir, 'build-support', 'bin', 'thrift', *PLATMAP[platform])
def run_thrifts(self):
for src in self.target.sources:
if not self._run_thrift(src):
raise PythonThriftBuilder.CodeGenerationException(
"Could not generate .py from %s!" % src)
def _run_thrift(self, source):
thrift_file = source
thrift_abs_path = os.path.join(self.root, self.target.target_base, thrift_file)
thrift_abs_path = os.path.abspath(thrift_abs_path)
args = [
PythonThriftBuilder.thrift_binary(self.root),
'--gen',
'py:new_style',
'-o',
self.codegen_root,
thrift_abs_path]
cwd = os.getcwd()
os.chdir(self.chroot.path())
print 'PythonThriftBuilder executing: %s' % ' '.join(args)
try:
po = subprocess.Popen(args)
finally:
os.chdir(cwd)
rv = po.wait()
if rv != 0:
comm = po.communicate()
print >> sys.stderr, 'thrift generation failed!'
print >> sys.stderr, 'STDOUT'
print >> sys.stderr, comm[0]
print >> sys.stderr, 'STDERR'
print >> sys.stderr, comm[1]
return rv == 0
@staticmethod
def path_to_module(path):
return path.replace(os.path.sep, '.')
def build_egg(self):
# autogenerate the python files that we bundle up
self.run_thrifts()
genpy_root = os.path.join(self.chroot.path(), self.codegen_root, 'gen-py')
for dir, subdirs, files in os.walk(os.path.normpath(genpy_root)):
reldir = os.path.relpath(dir, genpy_root)
if reldir == '.': continue
if '__init__.py' not in files: continue
init_py_abspath = os.path.join(dir, '__init__.py')
module_path = self.path_to_module(reldir)
self.detected_packages.add(module_path)
# A namespace package is one that is just a container for other
# modules and subpackages. Setting their __init__.py files as follows
# allows them to be distributed across multiple eggs. Without this you
# couldn't have this egg share any package prefix with any other module
# in any other egg or in the source tree.
#
# Note that the thrift compiler should always generate empty __init__.py
# files, but we test for this anyway, just in case that changes.
if len(files) == 1 and os.path.getsize(init_py_abspath) == 0:
with open(init_py_abspath, 'w') as f:
f.write("__import__('pkg_resources').declare_namespace(__name__)")
self.detected_namespace_packages.add(module_path)
if not self.detected_packages:
raise PythonThriftBuilder.CodeGenerationException(
'No Thrift structures declared in %s!' % self.target)
def dump_setup_py(packages, namespace_packages):
boilerplate = """
from setuptools import setup
setup(name = "%(target_name)s",
version = "dev",
description = "autogenerated thrift bindings for %(target_name)s",
package_dir = { "": "gen-py" },
packages = %(packages)s,
namespace_packages = %(namespace_packages)s)
"""
boilerplate = boilerplate % {
'target_name': self.target.name,
'genpy_root': genpy_root,
'packages': repr(list(packages)),
'namespace_packages': repr(list(namespace_packages))
}
self.chroot.write(boilerplate, os.path.join(self.codegen_root, 'setup.py'))
dump_setup_py(self.detected_packages, self.detected_namespace_packages)
egg_root = os.path.join(self.chroot.path(), self.codegen_root)
egg_path = EggBuilder().build_egg(egg_root, self.target)
return egg_path
