def __init__(self, target, root_dir): self.target = target self.root = root_dir self.module = target.module distdir = os.path.join(self.root, 'dist') self.chroot = RelativeChroot(root_dir, distdir, target.name) codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.') self.codegen_root = os.path.relpath(codegen_root, self.chroot.path()) self.created_packages = set() self.created_namespace_packages = set()
def __init__(self, workdir, target, root_dir, target_suffix=None):
self.target = target
self.suffix = target_suffix or ''
self.root = root_dir
self.chroot = RelativeChroot(root_dir,
os.path.join(workdir,
'codegen'), target.name)
codegen_root = safe_mkdtemp(dir=self.chroot.path(), prefix='codegen.')
self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
self.created_packages = set()
self.created_namespace_packages = set()
def __init__(self, target, root_dir, config, target_suffix=None):
self.target = target
self.config = config
self.suffix = target_suffix or ''
self.root = root_dir
distdir = self.config.getdefault('pants_distdir')
self.chroot = RelativeChroot(root_dir, distdir, target.name)
codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.')
self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
self.created_packages = set()
self.created_namespace_packages = set()
def __init__(self, target, root_dir, options, target_suffix=None):
self.target = target
self.options = options
self.suffix = target_suffix or ''
self.root = root_dir
self.chroot = RelativeChroot(root_dir,
options.for_global_scope().pants_distdir,
target.name)
codegen_root = tempfile.mkdtemp(dir=self.chroot.path(),
prefix='codegen.')
self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
self.created_packages = set()
self.created_namespace_packages = set()
def __init__(self, target, root_dir):
self.target = target
self.root = root_dir
distdir = os.path.join(self.root, 'dist')
self.chroot = RelativeChroot(root_dir, distdir, target.name)
codegen_root = tempfile.mkdtemp(dir=self.chroot.path(),
prefix='codegen.')
self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
self.detected_packages = set()
self.detected_namespace_packages = set()
# TODO: Temporary hack where we reparse pants.ini. Right now it's too difficult to plumb
# it through, and this will all be ported to "new pants" soon anyway.
config = Config.load()
self.platmap = config.getdict('py', 'thrift-platmap')
def __init__(self, workdir, target, root_dir, target_suffix=None):
self.target = target
self.suffix = target_suffix or ""
self.root = root_dir
self.chroot = RelativeChroot(root_dir, os.path.join(workdir, "codegen"), target.name)
codegen_root = safe_mkdtemp(dir=self.chroot.path(), prefix="codegen.")
self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
self.created_packages = set()
self.created_namespace_packages = set()
def __init__(self, target, root_dir): self.target = target self.root = root_dir distdir = os.path.join(self.root, 'dist') self.chroot = RelativeChroot(root_dir, distdir, target.name) codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.') self.codegen_root = os.path.relpath(codegen_root, self.chroot.path()) self.detected_packages = set() self.detected_namespace_packages = set()
def __init__(self, target, root_dir, options, target_suffix=None): self.target = target self.options = options self.suffix = target_suffix or '' self.root = root_dir self.chroot = RelativeChroot(root_dir, options.for_global_scope().pants_distdir, target.name) codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.') self.codegen_root = os.path.relpath(codegen_root, self.chroot.path()) self.created_packages = set() self.created_namespace_packages = set()
def __init__(self, target, root_dir):
self.target = target
self.root = root_dir
distdir = os.path.join(self.root, 'dist')
self.chroot = RelativeChroot(root_dir, distdir, target.name)
codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.')
self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
self.detected_packages = set()
self.detected_namespace_packages = set()
# TODO: Temporary hack where we reparse pants.ini. Right now it's too difficult to plumb
# it through, and this will all be ported to "new pants" soon anyway.
config = Config.load()
self.platmap = config.getdict('py', 'thrift-platmap')
class PythonThriftBuilder(object):
"""
Thrift builder.
"""
class UnknownPlatformException(Exception):
def __init__(self, platform):
Exception.__init__(self, "Unknown platform: %s!" % str(platform))
class CodeGenerationException(Exception): pass
def __init__(self, target, root_dir):
self.target = target
self.root = root_dir
distdir = os.path.join(self.root, 'dist')
self.chroot = RelativeChroot(root_dir, distdir, target.name)
codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.')
self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
self.detected_packages = set()
self.detected_namespace_packages = set()
def __del__(self):
self.cleanup()
def packages(self):
return self.detected_packages
def cleanup(self):
safe_rmtree(self.chroot.path())
@staticmethod
def thrift_binary(root_dir):
# TODO(Brian Wickman): Will we ever need to make this smarter?
PLATMAP = {
('darwin', 'i386'): ['mac', '10.6', '0.5.0-finagle', 'thrift'],
('darwin', 'x86_64'): ['mac', '10.6', '0.5.0-finagle', 'thrift'],
('darwin', 'x86_64'): ['mac', '10.7', '0.5.0-finagle', 'thrift'],
('linux', 'x86_64'): ['linux', 'x86_64', '0.5.0-finagle', 'thrift'],
('linux', 'i686') : ['linux', 'i386', '0.5.0-finagle', 'thrift']
}
uname = os.uname()
platform = (uname[0].lower(), uname[4])
if platform not in PLATMAP:
raise PythonThriftBuilder.UnknownPlatformException(platform)
return os.path.join(root_dir, 'build-support', 'bin', 'thrift', *PLATMAP[platform])
def run_thrifts(self):
for src in self.target.sources:
if not self._run_thrift(src):
raise PythonThriftBuilder.CodeGenerationException(
"Could not generate .py from %s!" % src)
def _run_thrift(self, source):
thrift_file = source
thrift_abs_path = os.path.join(self.root, self.target.target_base, thrift_file)
thrift_abs_path = os.path.abspath(thrift_abs_path)
args = [
PythonThriftBuilder.thrift_binary(self.root),
'--gen',
'py:new_style',
'-o',
self.codegen_root,
thrift_abs_path]
cwd = os.getcwd()
os.chdir(self.chroot.path())
try:
po = subprocess.Popen(args)
finally:
os.chdir(cwd)
rv = po.wait()
if rv != 0:
comm = po.communicate()
print('thrift generation failed!', file=sys.stderr)
print('STDOUT', file=sys.stderr)
print(comm[0], file=sys.stderr)
print('STDERR', file=sys.stderr)
print(comm[1], file=sys.stderr)
return rv == 0
@staticmethod
def path_to_module(path):
return path.replace(os.path.sep, '.')
def build_egg(self):
# autogenerate the python files that we bundle up
self.run_thrifts()
genpy_root = os.path.join(self.chroot.path(), self.codegen_root, 'gen-py')
for dir, _, files in os.walk(os.path.normpath(genpy_root)):
reldir = os.path.relpath(dir, genpy_root)
if reldir == '.': continue
if '__init__.py' not in files: continue
init_py_abspath = os.path.join(dir, '__init__.py')
module_path = self.path_to_module(reldir)
self.detected_packages.add(module_path)
# A namespace package is one that is just a container for other
# modules and subpackages. Setting their __init__.py files as follows
# allows them to be distributed across multiple eggs. Without this you
# couldn't have this egg share any package prefix with any other module
# in any other egg or in the source tree.
#
# Note that the thrift compiler should always generate empty __init__.py
# files, but we test for this anyway, just in case that changes.
if len(files) == 1 and os.path.getsize(init_py_abspath) == 0:
with open(init_py_abspath, 'wb') as f:
f.write(b"__import__('pkg_resources').declare_namespace(__name__)")
self.detected_namespace_packages.add(module_path)
if not self.detected_packages:
raise PythonThriftBuilder.CodeGenerationException(
'No Thrift structures declared in %s!' % self.target)
def dump_setup_py(packages, namespace_packages):
boilerplate = """
from setuptools import setup
setup(name = "%(target_name)s",
version = "dev",
description = "autogenerated thrift bindings for %(target_name)s",
package_dir = { "": "gen-py" },
packages = %(packages)s,
namespace_packages = %(namespace_packages)s)
"""
boilerplate = boilerplate % {
'target_name': self.target._create_id(),
'genpy_root': genpy_root,
'packages': repr(list(packages)),
'namespace_packages': repr(list(namespace_packages))
}
self.chroot.write(boilerplate.encode('utf-8'), os.path.join(self.codegen_root, 'setup.py'))
dump_setup_py(self.detected_packages, self.detected_namespace_packages)
egg_root = os.path.join(self.chroot.path(), self.codegen_root)
egg_path = EggBuilder().build_egg(egg_root, self.target)
return egg_path
class CodeGenerator(object):
class Error(Exception):
pass
class CodeGenerationException(Error):
pass
def __init__(self, workdir, target, root_dir, target_suffix=None):
self.target = target
self.suffix = target_suffix or ""
self.root = root_dir
self.chroot = RelativeChroot(root_dir, os.path.join(workdir, "codegen"), target.name)
codegen_root = safe_mkdtemp(dir=self.chroot.path(), prefix="codegen.")
self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
self.created_packages = set()
self.created_namespace_packages = set()
def __del__(self):
self.cleanup()
def cleanup(self):
shutil.rmtree(self.chroot.path())
@staticmethod
def path_to_module(path):
return path.replace(os.path.sep, ".")
def package_name(self):
return "{}{}".format(self.target.id, self.suffix)
def requirement_string(self):
return "{}==0.0.0".format(self.package_name())
@property
def package_dir(self):
"""Return the code generation root."""
return "."
@property
def install_requires(self):
return []
def generate(self):
"""Generate code for this target, updating the sets .created_packages and
.created_namespace_packages."""
raise NotImplementedError
def dump_setup_py(self):
boilerplate = textwrap.dedent(
"""
from setuptools import setup
setup(name = "{package_name}",
version = "0.0.0",
description = "autogenerated code for {target_name}",
install_requires = {install_requires!r},
package_dir = {{ "": {package_dir!r} }},
packages = {packages},
namespace_packages = {namespace_packages})
"""
)
boilerplate = boilerplate.format(
package_name=self.package_name().encode("utf-8"),
package_dir=self.package_dir.encode("utf-8"),
target_name=self.target.name.encode("utf-8"),
install_requires=[x.encode("utf-8") for x in self.install_requires],
packages=repr([x.encode("utf-8") for x in self.created_packages]),
namespace_packages=repr([x.encode("utf-8") for x in self.created_namespace_packages]),
)
self.chroot.write(boilerplate.encode("utf8"), os.path.join(self.codegen_root, "setup.py"))
self.chroot.write("include *.py".encode("utf8"), os.path.join(self.codegen_root, "MANIFEST.in"))
@property
def sdist_root(self):
return os.path.join(self.chroot.path(), self.codegen_root)
@property
def package_root(self):
return os.path.join(self.sdist_root, self.package_dir)
def build(self, interpreter=None):
self.generate()
self.dump_setup_py()
return SdistBuilder.build(self.sdist_root, self.target, interpreter=interpreter)
class CodeGenerator(object):
class Error(Exception): pass
class CodeGenerationException(Error): pass
def __init__(self, target, root_dir, config, target_suffix=None):
self.target = target
self.config = config
self.suffix = target_suffix or ''
self.root = root_dir
distdir = self.config.getdefault('pants_distdir')
self.chroot = RelativeChroot(root_dir, distdir, target.name)
codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.')
self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
self.created_packages = set()
self.created_namespace_packages = set()
def __del__(self):
self.cleanup()
def cleanup(self):
shutil.rmtree(self.chroot.path())
@staticmethod
def path_to_module(path):
return path.replace(os.path.sep, '.')
def package_name(self):
return '%s%s' % (self.target.id, self.suffix)
def requirement_string(self):
return '%s==0.0.0' % self.package_name()
@property
def package_dir(self):
"""Return the code generation root."""
return '.'
@property
def install_requires(self):
return []
def generate(self):
"""Generate code for this target, updating the sets .created_packages and
.created_namespace_packages."""
raise NotImplementedError
def dump_setup_py(self):
boilerplate = textwrap.dedent("""
from setuptools import setup
setup(name = "%(package_name)s",
version = "0.0.0",
description = "autogenerated code for %(target_name)s",
install_requires = %(install_requires)r,
package_dir = { "": %(package_dir)r },
packages = %(packages)s,
namespace_packages = %(namespace_packages)s)
""")
boilerplate = boilerplate % {
'package_name': self.package_name().encode('utf-8'),
'package_dir': self.package_dir.encode('utf-8'),
'target_name': self.target.name.encode('utf-8'),
'install_requires': [x.encode('utf-8') for x in self.install_requires],
'packages': repr([x.encode('utf-8') for x in self.created_packages]),
'namespace_packages': repr([x.encode('utf-8') for x in self.created_namespace_packages])
}
self.chroot.write(boilerplate.encode('utf8'), os.path.join(self.codegen_root, 'setup.py'))
self.chroot.write('include *.py'.encode('utf8'), os.path.join(self.codegen_root, 'MANIFEST.in'))
@property
def sdist_root(self):
return os.path.join(self.chroot.path(), self.codegen_root)
@property
def package_root(self):
return os.path.join(self.sdist_root, self.package_dir)
def build(self, interpreter=None):
self.generate()
self.dump_setup_py()
return SdistBuilder.build(self.sdist_root, self.target, interpreter=interpreter)
class CodeGenerator(object):
class Error(Exception):
pass
class CodeGenerationException(Error):
pass
def __init__(self, workdir, target, root_dir, target_suffix=None):
self.target = target
self.suffix = target_suffix or ''
self.root = root_dir
self.chroot = RelativeChroot(root_dir,
os.path.join(workdir,
'codegen'), target.name)
codegen_root = safe_mkdtemp(dir=self.chroot.path(), prefix='codegen.')
self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
self.created_packages = set()
self.created_namespace_packages = set()
def __del__(self):
self.cleanup()
def cleanup(self):
shutil.rmtree(self.chroot.path())
@staticmethod
def path_to_module(path):
return path.replace(os.path.sep, '.')
def package_name(self):
return '{}{}'.format(self.target.id, self.suffix)
def requirement_string(self):
return '{}==0.0.0'.format(self.package_name())
@property
def package_dir(self):
"""Return the code generation root."""
return '.'
@property
def install_requires(self):
return []
def generate(self):
"""Generate code for this target, updating the sets .created_packages and
.created_namespace_packages."""
raise NotImplementedError
def dump_setup_py(self):
boilerplate = textwrap.dedent("""
from setuptools import setup
setup(name = "{package_name}",
version = "0.0.0",
description = "autogenerated code for {target_name}",
install_requires = {install_requires!r},
package_dir = {{ "": {package_dir!r} }},
packages = {packages},
namespace_packages = {namespace_packages})
""")
boilerplate = boilerplate.format(
package_name=self.package_name().encode('utf-8'),
package_dir=self.package_dir.encode('utf-8'),
target_name=self.target.name.encode('utf-8'),
install_requires=[
x.encode('utf-8') for x in self.install_requires
],
packages=repr([x.encode('utf-8') for x in self.created_packages]),
namespace_packages=repr(
[x.encode('utf-8') for x in self.created_namespace_packages]))
self.chroot.write(boilerplate.encode('utf8'),
os.path.join(self.codegen_root, 'setup.py'))
self.chroot.write('include *.py'.encode('utf8'),
os.path.join(self.codegen_root, 'MANIFEST.in'))
@property
def sdist_root(self):
return os.path.join(self.chroot.path(), self.codegen_root)
@property
def package_root(self):
return os.path.join(self.sdist_root, self.package_dir)
def build(self, interpreter=None):
self.generate()
self.dump_setup_py()
return SdistBuilder.build(self.sdist_root,
self.target,
interpreter=interpreter)
class PythonAntlrBuilder(object):
"""
Antlr builder.
"""
class CodeGenerationException(Exception): pass
def __init__(self, target, root_dir):
self.target = target
self.root = root_dir
self.module = target.module
distdir = os.path.join(self.root, 'dist')
self.chroot = RelativeChroot(root_dir, distdir, target.name)
codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.')
self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
self.created_packages = set()
self.created_namespace_packages = set()
def __del__(self):
self.cleanup()
def packages(self):
return self.created_packages
def cleanup(self):
shutil.rmtree(self.chroot.path())
def run_antlrs(self, output_dir):
args = [
'java',
'-jar',
os.path.join(self.root, 'build-support/ivy/lib/ivy-2.2.0.jar'),
'-settings',
os.path.join(self.root, 'build-support/ivy/ivysettings.xml'),
'-dependency',
'org.antlr',
'antlr',
self.target.antlr_version,
'-types',
'jar',
'-main',
'org.antlr.Tool',
'--',
'-fo',
output_dir]
for source in self.target.sources:
abs_path = os.path.abspath(os.path.join(self.root, self.target.target_base, source))
args.append(abs_path)
cwd = os.getcwd()
os.chdir(self.chroot.path())
print('PythonAntlrBuilder executing: %s' % ' '.join(args))
try:
po = subprocess.Popen(args)
finally:
os.chdir(cwd)
rv = po.wait()
if rv != 0:
comm = po.communicate()
print('ANTLR generation failed!', file=sys.stderr)
print('STDOUT', file=sys.stderr)
print(comm[0], file=sys.stderr)
print('STDERR', file=sys.stderr)
print(comm[1], file=sys.stderr)
return rv == 0
@staticmethod
def path_to_module(path):
return path.replace(os.path.sep, '.')
def build_egg(self):
gen_root = os.path.join(self.chroot.path(), self.codegen_root)
# Create the package structure.
path = gen_root
package = ''
for module_name in self.module.split('.'):
path = os.path.join(path, module_name)
if package == '':
package = module_name
else:
package = package + '.' + module_name
os.mkdir(path)
with open(os.path.join(path, '__init__.py'), 'w') as f:
if package != self.module: # Only write this in the non-leaf modules.
f.write("__import__('pkg_resources').declare_namespace(__name__)")
self.created_namespace_packages.add(package)
self.created_packages.add(package)
# autogenerate the python files that we bundle up
self.run_antlrs(path)
def dump_setup_py(packages, namespace_packages):
boilerplate = """
from setuptools import setup
setup(name = "%(target_name)s",
version = "dev",
description = "autogenerated ANTLR parsers for %(target_name)s",
package_dir = { "": "." },
packages = %(packages)s,
namespace_packages = %(namespace_packages)s)
"""
boilerplate = boilerplate % {
'target_name': self.target.name,
'packages': repr(packages),
'namespace_packages': repr(list(namespace_packages))
}
self.chroot.write(boilerplate, os.path.join(self.codegen_root, 'setup.py'))
dump_setup_py(self.created_packages, self.created_namespace_packages)
egg_root = os.path.join(self.chroot.path(), self.codegen_root)
egg_path = EggBuilder().build_egg(egg_root, self.target)
return egg_path
class PythonThriftBuilder(object):
"""
Thrift builder.
"""
class UnknownPlatformException(Exception):
def __init__(self, platform):
Exception.__init__(self, "Unknown platform: %s!" % str(platform))
class CodeGenerationException(Exception):
pass
def __init__(self, target, root_dir, config):
self.target = target
self.root = root_dir
self.config = config
distdir = os.path.join(self.root, 'dist')
self.chroot = RelativeChroot(root_dir, distdir, target.name)
codegen_root = tempfile.mkdtemp(dir=self.chroot.path(),
prefix='codegen.')
self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
self.detected_packages = set()
self.detected_namespace_packages = set()
def __del__(self):
self.cleanup()
def packages(self):
return self.detected_packages
def cleanup(self):
safe_rmtree(self.chroot.path())
def run_thrifts(self):
def is_py_thrift(target):
return isinstance(target, PythonThriftLibrary)
bases, roots = calculate_compile_roots([self.target], is_py_thrift)
for src in roots:
if not self._run_thrift(src, bases):
raise PythonThriftBuilder.CodeGenerationException(
"Could not generate .py from %s!" % src)
def _run_thrift(self, source, bases):
thrift_file = source
thrift_abs_path = os.path.join(self.root, thrift_file)
thrift_abs_path = os.path.abspath(thrift_abs_path)
args = [
select_thrift_binary(self.config), '--gen', 'py:new_style',
'-recurse', '-o',
os.path.join(self.chroot.path(), self.codegen_root)
]
for base in bases:
args.extend(('-I', base))
args.append(thrift_abs_path)
po = subprocess.Popen(args)
rv = po.wait()
if rv != 0:
comm = po.communicate()
print('thrift generation failed!', file=sys.stderr)
print('STDOUT', file=sys.stderr)
print(comm[0], file=sys.stderr)
print('STDERR', file=sys.stderr)
print(comm[1], file=sys.stderr)
return rv == 0
@staticmethod
def path_to_module(path):
return path.replace(os.path.sep, '.')
def build_egg(self):
# autogenerate the python files that we bundle up
self.run_thrifts()
genpy_root = os.path.join(self.chroot.path(), self.codegen_root,
'gen-py')
for dir, _, files in os.walk(os.path.normpath(genpy_root)):
reldir = os.path.relpath(dir, genpy_root)
if reldir == '.': continue
if '__init__.py' not in files: continue
init_py_abspath = os.path.join(dir, '__init__.py')
module_path = self.path_to_module(reldir)
self.detected_packages.add(module_path)
# A namespace package is one that is just a container for other
# modules and subpackages. Setting their __init__.py files as follows
# allows them to be distributed across multiple eggs. Without this you
# couldn't have this egg share any package prefix with any other module
# in any other egg or in the source tree.
#
# Note that the thrift compiler should always generate empty __init__.py
# files, but we test for this anyway, just in case that changes.
if len(files) == 1 and os.path.getsize(init_py_abspath) == 0:
with open(init_py_abspath, 'wb') as f:
f.write(
b"__import__('pkg_resources').declare_namespace(__name__)"
)
self.detected_namespace_packages.add(module_path)
if not self.detected_packages:
raise PythonThriftBuilder.CodeGenerationException(
'No Thrift structures declared in %s!' % self.target)
def dump_setup_py(packages, namespace_packages):
boilerplate = """
from setuptools import setup
setup(name = "%(target_name)s",
version = "dev",
description = "autogenerated thrift bindings for %(target_name)s",
package_dir = { "": "gen-py" },
packages = %(packages)s,
namespace_packages = %(namespace_packages)s)
"""
boilerplate = boilerplate % {
'target_name': self.target._create_id(),
'genpy_root': genpy_root,
'packages': repr(list(packages)),
'namespace_packages': repr(list(namespace_packages))
}
self.chroot.write(boilerplate.encode('utf-8'),
os.path.join(self.codegen_root, 'setup.py'))
dump_setup_py(self.detected_packages, self.detected_namespace_packages)
egg_root = os.path.join(self.chroot.path(), self.codegen_root)
egg_path = EggBuilder().build_egg(egg_root, self.target)
return egg_path
class PythonThriftBuilder(object):
"""
Thrift builder.
"""
class UnknownPlatformException(Exception):
def __init__(self, platform):
Exception.__init__(self, "Unknown platform: %s!" % str(platform))
class CodeGenerationException(Exception):
pass
def __init__(self, target, root_dir):
self.target = target
self.root = root_dir
distdir = os.path.join(self.root, 'dist')
self.chroot = RelativeChroot(root_dir, distdir, target.name)
codegen_root = tempfile.mkdtemp(dir=self.chroot.path(),
prefix='codegen.')
self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
self.detected_packages = set()
self.detected_namespace_packages = set()
def __del__(self):
self.cleanup()
def packages(self):
return self.detected_packages
def cleanup(self):
safe_rmtree(self.chroot.path())
@staticmethod
def thrift_binary(root_dir):
# TODO(Brian Wickman): Will we ever need to make this smarter?
PLATMAP = {
('darwin', 'i386'): ['mac', '10.6', '0.5.0-finagle', 'thrift'],
('darwin', 'x86_64'): ['mac', '10.6', '0.5.0-finagle', 'thrift'],
('darwin', 'x86_64'): ['mac', '10.7', '0.5.0-finagle', 'thrift'],
('linux', 'x86_64'):
['linux', 'x86_64', '0.5.0-finagle', 'thrift'],
('linux', 'i686'): ['linux', 'i386', '0.5.0-finagle', 'thrift']
}
uname = os.uname()
platform = (uname[0].lower(), uname[4])
if platform not in PLATMAP:
raise PythonThriftBuilder.UnknownPlatformException(platform)
return os.path.join(root_dir, 'build-support', 'bin', 'thrift',
*PLATMAP[platform])
def run_thrifts(self):
for src in self.target.sources:
if not self._run_thrift(src):
raise PythonThriftBuilder.CodeGenerationException(
"Could not generate .py from %s!" % src)
def _run_thrift(self, source):
thrift_file = source
thrift_abs_path = os.path.join(self.root, self.target.target_base,
thrift_file)
thrift_abs_path = os.path.abspath(thrift_abs_path)
args = [
PythonThriftBuilder.thrift_binary(self.root), '--gen',
'py:new_style', '-o', self.codegen_root, thrift_abs_path
]
cwd = os.getcwd()
os.chdir(self.chroot.path())
try:
po = subprocess.Popen(args)
finally:
os.chdir(cwd)
rv = po.wait()
if rv != 0:
comm = po.communicate()
print('thrift generation failed!', file=sys.stderr)
print('STDOUT', file=sys.stderr)
print(comm[0], file=sys.stderr)
print('STDERR', file=sys.stderr)
print(comm[1], file=sys.stderr)
return rv == 0
@staticmethod
def path_to_module(path):
return path.replace(os.path.sep, '.')
def build_egg(self):
# autogenerate the python files that we bundle up
self.run_thrifts()
genpy_root = os.path.join(self.chroot.path(), self.codegen_root,
'gen-py')
for dir, _, files in os.walk(os.path.normpath(genpy_root)):
reldir = os.path.relpath(dir, genpy_root)
if reldir == '.': continue
if '__init__.py' not in files: continue
init_py_abspath = os.path.join(dir, '__init__.py')
module_path = self.path_to_module(reldir)
self.detected_packages.add(module_path)
# A namespace package is one that is just a container for other
# modules and subpackages. Setting their __init__.py files as follows
# allows them to be distributed across multiple eggs. Without this you
# couldn't have this egg share any package prefix with any other module
# in any other egg or in the source tree.
#
# Note that the thrift compiler should always generate empty __init__.py
# files, but we test for this anyway, just in case that changes.
if len(files) == 1 and os.path.getsize(init_py_abspath) == 0:
with open(init_py_abspath, 'wb') as f:
f.write(
b"__import__('pkg_resources').declare_namespace(__name__)"
)
self.detected_namespace_packages.add(module_path)
if not self.detected_packages:
raise PythonThriftBuilder.CodeGenerationException(
'No Thrift structures declared in %s!' % self.target)
def dump_setup_py(packages, namespace_packages):
boilerplate = """
from setuptools import setup
setup(name = "%(target_name)s",
version = "dev",
description = "autogenerated thrift bindings for %(target_name)s",
package_dir = { "": "gen-py" },
packages = %(packages)s,
namespace_packages = %(namespace_packages)s)
"""
boilerplate = boilerplate % {
'target_name': self.target._create_id(),
'genpy_root': genpy_root,
'packages': repr(list(packages)),
'namespace_packages': repr(list(namespace_packages))
}
self.chroot.write(boilerplate.encode('utf-8'),
os.path.join(self.codegen_root, 'setup.py'))
dump_setup_py(self.detected_packages, self.detected_namespace_packages)
egg_root = os.path.join(self.chroot.path(), self.codegen_root)
egg_path = EggBuilder().build_egg(egg_root, self.target)
return egg_path
class PythonThriftBuilder(object):
"""
Thrift builder.
"""
class UnknownPlatformException(Exception):
def __init__(self, platform):
Exception.__init__(self, "Unknown platform: %s!" % str(platform))
class CodeGenerationException(Exception): pass
def __init__(self, target, root_dir, config):
self.target = target
self.root = root_dir
self.config = config
distdir = os.path.join(self.root, 'dist')
self.chroot = RelativeChroot(root_dir, distdir, target.name)
codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.')
self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
self.detected_packages = set()
self.detected_namespace_packages = set()
def __del__(self):
self.cleanup()
def packages(self):
return self.detected_packages
def cleanup(self):
safe_rmtree(self.chroot.path())
def run_thrifts(self):
def is_py_thrift(target):
return isinstance(target, PythonThriftLibrary)
bases, roots = calculate_compile_roots([self.target], is_py_thrift)
for src in roots:
if not self._run_thrift(src, bases):
raise PythonThriftBuilder.CodeGenerationException(
"Could not generate .py from %s!" % src)
def _run_thrift(self, source, bases):
thrift_file = source
thrift_abs_path = os.path.join(self.root, thrift_file)
thrift_abs_path = os.path.abspath(thrift_abs_path)
args = [
select_thrift_binary(self.config),
'--gen',
'py:new_style,utf8strings',
'-recurse',
'-o',
os.path.join(self.chroot.path(), self.codegen_root)
]
for base in bases:
args.extend(('-I', base))
args.append(thrift_abs_path)
po = subprocess.Popen(args)
rv = po.wait()
if rv != 0:
comm = po.communicate()
print('thrift generation failed!', file=sys.stderr)
print('STDOUT', file=sys.stderr)
print(comm[0], file=sys.stderr)
print('STDERR', file=sys.stderr)
print(comm[1], file=sys.stderr)
return rv == 0
@staticmethod
def path_to_module(path):
return path.replace(os.path.sep, '.')
def build_egg(self):
# autogenerate the python files that we bundle up
self.run_thrifts()
genpy_root = os.path.join(self.chroot.path(), self.codegen_root, 'gen-py')
for dir, _, files in os.walk(os.path.normpath(genpy_root)):
reldir = os.path.relpath(dir, genpy_root)
if reldir == '.': continue
if '__init__.py' not in files: continue
init_py_abspath = os.path.join(dir, '__init__.py')
module_path = self.path_to_module(reldir)
self.detected_packages.add(module_path)
# A namespace package is one that is just a container for other
# modules and subpackages. Setting their __init__.py files as follows
# allows them to be distributed across multiple eggs. Without this you
# couldn't have this egg share any package prefix with any other module
# in any other egg or in the source tree.
#
# Note that the thrift compiler should always generate empty __init__.py
# files, but we test for this anyway, just in case that changes.
if len(files) == 1 and os.path.getsize(init_py_abspath) == 0:
with open(init_py_abspath, 'wb') as f:
f.write(b"__import__('pkg_resources').declare_namespace(__name__)")
self.detected_namespace_packages.add(module_path)
if not self.detected_packages:
raise PythonThriftBuilder.CodeGenerationException(
'No Thrift structures declared in %s!' % self.target)
def dump_setup_py(packages, namespace_packages):
boilerplate = """
from setuptools import setup
setup(name = "%(target_name)s",
version = "dev",
description = "autogenerated thrift bindings for %(target_name)s",
package_dir = { "": "gen-py" },
packages = %(packages)s,
namespace_packages = %(namespace_packages)s)
"""
boilerplate = boilerplate % {
'target_name': self.target._create_id(),
'genpy_root': genpy_root,
'packages': repr(list(packages)),
'namespace_packages': repr(list(namespace_packages))
}
self.chroot.write(boilerplate.encode('utf-8'), os.path.join(self.codegen_root, 'setup.py'))
dump_setup_py(self.detected_packages, self.detected_namespace_packages)
egg_root = os.path.join(self.chroot.path(), self.codegen_root)
egg_path = EggBuilder().build_egg(egg_root, self.target)
return egg_path
