Esempio n. 1
0
 def __init__(self, target, root_dir):
   self.target = target
   self.root = root_dir
   self.module = target.module
   distdir = os.path.join(self.root, 'dist')
   self.chroot = RelativeChroot(root_dir, distdir, target.name)
   codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.')
   self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
   self.created_packages = set()
   self.created_namespace_packages = set()
Esempio n. 2
0
 def __init__(self, workdir, target, root_dir, target_suffix=None):
     self.target = target
     self.suffix = target_suffix or ''
     self.root = root_dir
     self.chroot = RelativeChroot(root_dir,
                                  os.path.join(workdir,
                                               'codegen'), target.name)
     codegen_root = safe_mkdtemp(dir=self.chroot.path(), prefix='codegen.')
     self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
     self.created_packages = set()
     self.created_namespace_packages = set()
Esempio n. 3
0
 def __init__(self, target, root_dir, config, target_suffix=None):
   self.target = target
   self.config = config
   self.suffix = target_suffix or ''
   self.root = root_dir
   distdir = self.config.getdefault('pants_distdir')
   self.chroot = RelativeChroot(root_dir, distdir, target.name)
   codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.')
   self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
   self.created_packages = set()
   self.created_namespace_packages = set()
Esempio n. 4
0
 def __init__(self, target, root_dir, options, target_suffix=None):
     self.target = target
     self.options = options
     self.suffix = target_suffix or ''
     self.root = root_dir
     self.chroot = RelativeChroot(root_dir,
                                  options.for_global_scope().pants_distdir,
                                  target.name)
     codegen_root = tempfile.mkdtemp(dir=self.chroot.path(),
                                     prefix='codegen.')
     self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
     self.created_packages = set()
     self.created_namespace_packages = set()
Esempio n. 5
0
    def __init__(self, target, root_dir):
        self.target = target
        self.root = root_dir
        distdir = os.path.join(self.root, 'dist')
        self.chroot = RelativeChroot(root_dir, distdir, target.name)
        codegen_root = tempfile.mkdtemp(dir=self.chroot.path(),
                                        prefix='codegen.')
        self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
        self.detected_packages = set()
        self.detected_namespace_packages = set()

        # TODO: Temporary hack where we reparse pants.ini. Right now it's too difficult to plumb
        # it through, and this will all be ported to "new pants" soon anyway.
        config = Config.load()
        self.platmap = config.getdict('py', 'thrift-platmap')
Esempio n. 6
0
 def __init__(self, workdir, target, root_dir, target_suffix=None):
     self.target = target
     self.suffix = target_suffix or ""
     self.root = root_dir
     self.chroot = RelativeChroot(root_dir, os.path.join(workdir, "codegen"), target.name)
     codegen_root = safe_mkdtemp(dir=self.chroot.path(), prefix="codegen.")
     self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
     self.created_packages = set()
     self.created_namespace_packages = set()
Esempio n. 7
0
 def __init__(self, target, root_dir):
   self.target = target
   self.root = root_dir
   distdir = os.path.join(self.root, 'dist')
   self.chroot = RelativeChroot(root_dir, distdir, target.name)
   codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.')
   self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
   self.detected_packages = set()
   self.detected_namespace_packages = set()
Esempio n. 8
0
 def __init__(self, target, root_dir, options, target_suffix=None):
   self.target = target
   self.options = options
   self.suffix = target_suffix or ''
   self.root = root_dir
   self.chroot = RelativeChroot(root_dir, options.for_global_scope().pants_distdir, target.name)
   codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.')
   self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
   self.created_packages = set()
   self.created_namespace_packages = set()
Esempio n. 9
0
  def __init__(self, target, root_dir):
    self.target = target
    self.root = root_dir
    distdir = os.path.join(self.root, 'dist')
    self.chroot = RelativeChroot(root_dir, distdir, target.name)
    codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.')
    self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
    self.detected_packages = set()
    self.detected_namespace_packages = set()

    # TODO: Temporary hack where we reparse pants.ini. Right now it's too difficult to plumb
    # it through, and this will all be ported to "new pants" soon anyway.
    config = Config.load()
    self.platmap = config.getdict('py', 'thrift-platmap')
Esempio n. 10
0
class PythonThriftBuilder(object):
  """
    Thrift builder.
  """
  class UnknownPlatformException(Exception):
    def __init__(self, platform):
      Exception.__init__(self, "Unknown platform: %s!" % str(platform))
  class CodeGenerationException(Exception): pass

  def __init__(self, target, root_dir):
    self.target = target
    self.root = root_dir
    distdir = os.path.join(self.root, 'dist')
    self.chroot = RelativeChroot(root_dir, distdir, target.name)
    codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.')
    self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
    self.detected_packages = set()
    self.detected_namespace_packages = set()

  def __del__(self):
    self.cleanup()

  def packages(self):
    return self.detected_packages

  def cleanup(self):
    safe_rmtree(self.chroot.path())

  @staticmethod
  def thrift_binary(root_dir):
    # TODO(Brian Wickman):  Will we ever need to make this smarter?
    PLATMAP = {
      ('darwin', 'i386'): ['mac', '10.6', '0.5.0-finagle', 'thrift'],
      ('darwin', 'x86_64'): ['mac', '10.6', '0.5.0-finagle', 'thrift'],
      ('darwin', 'x86_64'): ['mac', '10.7', '0.5.0-finagle', 'thrift'],
      ('linux',  'x86_64'): ['linux', 'x86_64', '0.5.0-finagle', 'thrift'],
      ('linux',  'i686') : ['linux', 'i386', '0.5.0-finagle', 'thrift']
    }
    uname = os.uname()
    platform = (uname[0].lower(), uname[4])
    if platform not in PLATMAP:
      raise PythonThriftBuilder.UnknownPlatformException(platform)
    return os.path.join(root_dir, 'build-support', 'bin', 'thrift', *PLATMAP[platform])

  def run_thrifts(self):
    for src in self.target.sources:
      if not self._run_thrift(src):
        raise PythonThriftBuilder.CodeGenerationException(
          "Could not generate .py from %s!" % src)

  def _run_thrift(self, source):
    thrift_file = source
    thrift_abs_path = os.path.join(self.root, self.target.target_base, thrift_file)
    thrift_abs_path = os.path.abspath(thrift_abs_path)

    args = [
      PythonThriftBuilder.thrift_binary(self.root),
      '--gen',
      'py:new_style',
      '-o',
      self.codegen_root,
      thrift_abs_path]

    cwd = os.getcwd()
    os.chdir(self.chroot.path())
    try:
      po = subprocess.Popen(args)
    finally:
      os.chdir(cwd)
    rv = po.wait()
    if rv != 0:
      comm = po.communicate()
      print('thrift generation failed!', file=sys.stderr)
      print('STDOUT', file=sys.stderr)
      print(comm[0], file=sys.stderr)
      print('STDERR', file=sys.stderr)
      print(comm[1], file=sys.stderr)
    return rv == 0

  @staticmethod
  def path_to_module(path):
    return path.replace(os.path.sep, '.')

  def build_egg(self):
    # autogenerate the python files that we bundle up
    self.run_thrifts()

    genpy_root = os.path.join(self.chroot.path(), self.codegen_root, 'gen-py')
    for dir, _, files in os.walk(os.path.normpath(genpy_root)):
      reldir = os.path.relpath(dir, genpy_root)
      if reldir == '.': continue
      if '__init__.py' not in files: continue
      init_py_abspath = os.path.join(dir, '__init__.py')
      module_path = self.path_to_module(reldir)
      self.detected_packages.add(module_path)
      # A namespace package is one that is just a container for other
      # modules and subpackages. Setting their __init__.py files as follows
      # allows them to be distributed across multiple eggs. Without this you
      # couldn't have this egg share any package prefix with any other module
      # in any other egg or in the source tree.
      #
      # Note that the thrift compiler should always generate empty __init__.py
      # files, but we test for this anyway, just in case that changes.
      if len(files) == 1 and os.path.getsize(init_py_abspath) == 0:
        with open(init_py_abspath, 'wb') as f:
          f.write(b"__import__('pkg_resources').declare_namespace(__name__)")
        self.detected_namespace_packages.add(module_path)

    if not self.detected_packages:
      raise PythonThriftBuilder.CodeGenerationException(
        'No Thrift structures declared in %s!' % self.target)

    def dump_setup_py(packages, namespace_packages):
      boilerplate = """
from setuptools import setup

setup(name        = "%(target_name)s",
      version     = "dev",
      description = "autogenerated thrift bindings for %(target_name)s",
      package_dir = { "": "gen-py" },
      packages    = %(packages)s,
      namespace_packages = %(namespace_packages)s)
"""
      boilerplate = boilerplate % {
        'target_name': self.target._create_id(),
        'genpy_root': genpy_root,
        'packages': repr(list(packages)),
        'namespace_packages': repr(list(namespace_packages))
      }

      self.chroot.write(boilerplate.encode('utf-8'), os.path.join(self.codegen_root, 'setup.py'))
    dump_setup_py(self.detected_packages, self.detected_namespace_packages)

    egg_root = os.path.join(self.chroot.path(), self.codegen_root)
    egg_path = EggBuilder().build_egg(egg_root, self.target)
    return egg_path
Esempio n. 11
0
class CodeGenerator(object):
    class Error(Exception):
        pass

    class CodeGenerationException(Error):
        pass

    def __init__(self, workdir, target, root_dir, target_suffix=None):
        self.target = target
        self.suffix = target_suffix or ""
        self.root = root_dir
        self.chroot = RelativeChroot(root_dir, os.path.join(workdir, "codegen"), target.name)
        codegen_root = safe_mkdtemp(dir=self.chroot.path(), prefix="codegen.")
        self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
        self.created_packages = set()
        self.created_namespace_packages = set()

    def __del__(self):
        self.cleanup()

    def cleanup(self):
        shutil.rmtree(self.chroot.path())

    @staticmethod
    def path_to_module(path):
        return path.replace(os.path.sep, ".")

    def package_name(self):
        return "{}{}".format(self.target.id, self.suffix)

    def requirement_string(self):
        return "{}==0.0.0".format(self.package_name())

    @property
    def package_dir(self):
        """Return the code generation root."""
        return "."

    @property
    def install_requires(self):
        return []

    def generate(self):
        """Generate code for this target, updating the sets .created_packages and
       .created_namespace_packages."""
        raise NotImplementedError

    def dump_setup_py(self):
        boilerplate = textwrap.dedent(
            """
      from setuptools import setup

      setup(name        = "{package_name}",
            version     = "0.0.0",
            description = "autogenerated code for {target_name}",
            install_requires = {install_requires!r},
            package_dir = {{ "": {package_dir!r} }},
            packages    = {packages},
            namespace_packages = {namespace_packages})
    """
        )
        boilerplate = boilerplate.format(
            package_name=self.package_name().encode("utf-8"),
            package_dir=self.package_dir.encode("utf-8"),
            target_name=self.target.name.encode("utf-8"),
            install_requires=[x.encode("utf-8") for x in self.install_requires],
            packages=repr([x.encode("utf-8") for x in self.created_packages]),
            namespace_packages=repr([x.encode("utf-8") for x in self.created_namespace_packages]),
        )
        self.chroot.write(boilerplate.encode("utf8"), os.path.join(self.codegen_root, "setup.py"))
        self.chroot.write("include *.py".encode("utf8"), os.path.join(self.codegen_root, "MANIFEST.in"))

    @property
    def sdist_root(self):
        return os.path.join(self.chroot.path(), self.codegen_root)

    @property
    def package_root(self):
        return os.path.join(self.sdist_root, self.package_dir)

    def build(self, interpreter=None):
        self.generate()
        self.dump_setup_py()
        return SdistBuilder.build(self.sdist_root, self.target, interpreter=interpreter)
Esempio n. 12
0
class CodeGenerator(object):
  class Error(Exception): pass
  class CodeGenerationException(Error): pass

  def __init__(self, target, root_dir, config, target_suffix=None):
    self.target = target
    self.config = config
    self.suffix = target_suffix or ''
    self.root = root_dir
    distdir = self.config.getdefault('pants_distdir')
    self.chroot = RelativeChroot(root_dir, distdir, target.name)
    codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.')
    self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
    self.created_packages = set()
    self.created_namespace_packages = set()

  def __del__(self):
    self.cleanup()

  def cleanup(self):
    shutil.rmtree(self.chroot.path())

  @staticmethod
  def path_to_module(path):
    return path.replace(os.path.sep, '.')

  def package_name(self):
    return '%s%s' % (self.target.id, self.suffix)

  def requirement_string(self):
    return '%s==0.0.0' % self.package_name()

  @property
  def package_dir(self):
    """Return the code generation root."""
    return '.'

  @property
  def install_requires(self):
    return []

  def generate(self):
    """Generate code for this target, updating the sets .created_packages and
       .created_namespace_packages."""
    raise NotImplementedError

  def dump_setup_py(self):
    boilerplate = textwrap.dedent("""
      from setuptools import setup

      setup(name        = "%(package_name)s",
            version     = "0.0.0",
            description = "autogenerated code for %(target_name)s",
            install_requires = %(install_requires)r,
            package_dir = { "": %(package_dir)r },
            packages    = %(packages)s,
            namespace_packages = %(namespace_packages)s)
    """)
    boilerplate = boilerplate % {
      'package_name': self.package_name().encode('utf-8'),
      'package_dir': self.package_dir.encode('utf-8'),
      'target_name': self.target.name.encode('utf-8'),
      'install_requires': [x.encode('utf-8') for x in self.install_requires],
      'packages': repr([x.encode('utf-8') for x in self.created_packages]),
      'namespace_packages': repr([x.encode('utf-8') for x in self.created_namespace_packages])
    }
    self.chroot.write(boilerplate.encode('utf8'), os.path.join(self.codegen_root, 'setup.py'))
    self.chroot.write('include *.py'.encode('utf8'), os.path.join(self.codegen_root, 'MANIFEST.in'))

  @property
  def sdist_root(self):
    return os.path.join(self.chroot.path(), self.codegen_root)

  @property
  def package_root(self):
    return os.path.join(self.sdist_root, self.package_dir)

  def build(self, interpreter=None):
    self.generate()
    self.dump_setup_py()
    return SdistBuilder.build(self.sdist_root, self.target, interpreter=interpreter)
Esempio n. 13
0
class CodeGenerator(object):
    class Error(Exception):
        pass

    class CodeGenerationException(Error):
        pass

    def __init__(self, workdir, target, root_dir, target_suffix=None):
        self.target = target
        self.suffix = target_suffix or ''
        self.root = root_dir
        self.chroot = RelativeChroot(root_dir,
                                     os.path.join(workdir,
                                                  'codegen'), target.name)
        codegen_root = safe_mkdtemp(dir=self.chroot.path(), prefix='codegen.')
        self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
        self.created_packages = set()
        self.created_namespace_packages = set()

    def __del__(self):
        self.cleanup()

    def cleanup(self):
        shutil.rmtree(self.chroot.path())

    @staticmethod
    def path_to_module(path):
        return path.replace(os.path.sep, '.')

    def package_name(self):
        return '{}{}'.format(self.target.id, self.suffix)

    def requirement_string(self):
        return '{}==0.0.0'.format(self.package_name())

    @property
    def package_dir(self):
        """Return the code generation root."""
        return '.'

    @property
    def install_requires(self):
        return []

    def generate(self):
        """Generate code for this target, updating the sets .created_packages and
       .created_namespace_packages."""
        raise NotImplementedError

    def dump_setup_py(self):
        boilerplate = textwrap.dedent("""
      from setuptools import setup

      setup(name        = "{package_name}",
            version     = "0.0.0",
            description = "autogenerated code for {target_name}",
            install_requires = {install_requires!r},
            package_dir = {{ "": {package_dir!r} }},
            packages    = {packages},
            namespace_packages = {namespace_packages})
    """)
        boilerplate = boilerplate.format(
            package_name=self.package_name().encode('utf-8'),
            package_dir=self.package_dir.encode('utf-8'),
            target_name=self.target.name.encode('utf-8'),
            install_requires=[
                x.encode('utf-8') for x in self.install_requires
            ],
            packages=repr([x.encode('utf-8') for x in self.created_packages]),
            namespace_packages=repr(
                [x.encode('utf-8') for x in self.created_namespace_packages]))
        self.chroot.write(boilerplate.encode('utf8'),
                          os.path.join(self.codegen_root, 'setup.py'))
        self.chroot.write('include *.py'.encode('utf8'),
                          os.path.join(self.codegen_root, 'MANIFEST.in'))

    @property
    def sdist_root(self):
        return os.path.join(self.chroot.path(), self.codegen_root)

    @property
    def package_root(self):
        return os.path.join(self.sdist_root, self.package_dir)

    def build(self, interpreter=None):
        self.generate()
        self.dump_setup_py()
        return SdistBuilder.build(self.sdist_root,
                                  self.target,
                                  interpreter=interpreter)
Esempio n. 14
0
class PythonAntlrBuilder(object):
  """
    Antlr builder.
  """
  class CodeGenerationException(Exception): pass

  def __init__(self, target, root_dir):
    self.target = target
    self.root = root_dir
    self.module = target.module
    distdir = os.path.join(self.root, 'dist')
    self.chroot = RelativeChroot(root_dir, distdir, target.name)
    codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.')
    self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
    self.created_packages = set()
    self.created_namespace_packages = set()

  def __del__(self):
    self.cleanup()

  def packages(self):
    return self.created_packages

  def cleanup(self):
    shutil.rmtree(self.chroot.path())

  def run_antlrs(self, output_dir):
    args = [
        'java',
        '-jar',
        os.path.join(self.root, 'build-support/ivy/lib/ivy-2.2.0.jar'),
        '-settings',
        os.path.join(self.root, 'build-support/ivy/ivysettings.xml'),
        '-dependency',
        'org.antlr',
        'antlr',
        self.target.antlr_version,
        '-types',
        'jar',
        '-main',
        'org.antlr.Tool',
        '--',
        '-fo',
        output_dir]
    for source in self.target.sources:
      abs_path = os.path.abspath(os.path.join(self.root, self.target.target_base, source))
      args.append(abs_path)

    cwd = os.getcwd()
    os.chdir(self.chroot.path())
    print('PythonAntlrBuilder executing: %s' % ' '.join(args))
    try:
      po = subprocess.Popen(args)
    finally:
      os.chdir(cwd)
    rv = po.wait()
    if rv != 0:
      comm = po.communicate()
      print('ANTLR generation failed!', file=sys.stderr)
      print('STDOUT', file=sys.stderr)
      print(comm[0], file=sys.stderr)
      print('STDERR', file=sys.stderr)
      print(comm[1], file=sys.stderr)
    return rv == 0

  @staticmethod
  def path_to_module(path):
    return path.replace(os.path.sep, '.')

  def build_egg(self):
    gen_root = os.path.join(self.chroot.path(), self.codegen_root)
    # Create the package structure.
    path = gen_root
    package = ''
    for module_name in self.module.split('.'):
      path = os.path.join(path, module_name)
      if package == '':
        package = module_name
      else:
        package = package + '.' + module_name
      os.mkdir(path)
      with open(os.path.join(path, '__init__.py'), 'w') as f:
        if package != self.module:  # Only write this in the non-leaf modules.
          f.write("__import__('pkg_resources').declare_namespace(__name__)")
          self.created_namespace_packages.add(package)
      self.created_packages.add(package)

    # autogenerate the python files that we bundle up
    self.run_antlrs(path)

    def dump_setup_py(packages, namespace_packages):
      boilerplate = """
from setuptools import setup

setup(name        = "%(target_name)s",
      version     = "dev",
      description = "autogenerated ANTLR parsers for %(target_name)s",
      package_dir = { "": "." },
      packages    = %(packages)s,
      namespace_packages = %(namespace_packages)s)
"""
      boilerplate = boilerplate % {
        'target_name': self.target.name,
        'packages': repr(packages),
        'namespace_packages': repr(list(namespace_packages))
      }

      self.chroot.write(boilerplate, os.path.join(self.codegen_root, 'setup.py'))
    dump_setup_py(self.created_packages, self.created_namespace_packages)

    egg_root = os.path.join(self.chroot.path(), self.codegen_root)
    egg_path = EggBuilder().build_egg(egg_root, self.target)
    return egg_path
Esempio n. 15
0
class PythonThriftBuilder(object):
    """
    Thrift builder.
  """
    class UnknownPlatformException(Exception):
        def __init__(self, platform):
            Exception.__init__(self, "Unknown platform: %s!" % str(platform))

    class CodeGenerationException(Exception):
        pass

    def __init__(self, target, root_dir, config):
        self.target = target
        self.root = root_dir
        self.config = config
        distdir = os.path.join(self.root, 'dist')
        self.chroot = RelativeChroot(root_dir, distdir, target.name)
        codegen_root = tempfile.mkdtemp(dir=self.chroot.path(),
                                        prefix='codegen.')
        self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
        self.detected_packages = set()
        self.detected_namespace_packages = set()

    def __del__(self):
        self.cleanup()

    def packages(self):
        return self.detected_packages

    def cleanup(self):
        safe_rmtree(self.chroot.path())

    def run_thrifts(self):
        def is_py_thrift(target):
            return isinstance(target, PythonThriftLibrary)

        bases, roots = calculate_compile_roots([self.target], is_py_thrift)

        for src in roots:
            if not self._run_thrift(src, bases):
                raise PythonThriftBuilder.CodeGenerationException(
                    "Could not generate .py from %s!" % src)

    def _run_thrift(self, source, bases):
        thrift_file = source
        thrift_abs_path = os.path.join(self.root, thrift_file)
        thrift_abs_path = os.path.abspath(thrift_abs_path)

        args = [
            select_thrift_binary(self.config), '--gen', 'py:new_style',
            '-recurse', '-o',
            os.path.join(self.chroot.path(), self.codegen_root)
        ]
        for base in bases:
            args.extend(('-I', base))
        args.append(thrift_abs_path)

        po = subprocess.Popen(args)
        rv = po.wait()
        if rv != 0:
            comm = po.communicate()
            print('thrift generation failed!', file=sys.stderr)
            print('STDOUT', file=sys.stderr)
            print(comm[0], file=sys.stderr)
            print('STDERR', file=sys.stderr)
            print(comm[1], file=sys.stderr)
        return rv == 0

    @staticmethod
    def path_to_module(path):
        return path.replace(os.path.sep, '.')

    def build_egg(self):
        # autogenerate the python files that we bundle up
        self.run_thrifts()

        genpy_root = os.path.join(self.chroot.path(), self.codegen_root,
                                  'gen-py')
        for dir, _, files in os.walk(os.path.normpath(genpy_root)):
            reldir = os.path.relpath(dir, genpy_root)
            if reldir == '.': continue
            if '__init__.py' not in files: continue
            init_py_abspath = os.path.join(dir, '__init__.py')
            module_path = self.path_to_module(reldir)
            self.detected_packages.add(module_path)
            # A namespace package is one that is just a container for other
            # modules and subpackages. Setting their __init__.py files as follows
            # allows them to be distributed across multiple eggs. Without this you
            # couldn't have this egg share any package prefix with any other module
            # in any other egg or in the source tree.
            #
            # Note that the thrift compiler should always generate empty __init__.py
            # files, but we test for this anyway, just in case that changes.
            if len(files) == 1 and os.path.getsize(init_py_abspath) == 0:
                with open(init_py_abspath, 'wb') as f:
                    f.write(
                        b"__import__('pkg_resources').declare_namespace(__name__)"
                    )
                self.detected_namespace_packages.add(module_path)

        if not self.detected_packages:
            raise PythonThriftBuilder.CodeGenerationException(
                'No Thrift structures declared in %s!' % self.target)

        def dump_setup_py(packages, namespace_packages):
            boilerplate = """
from setuptools import setup

setup(name        = "%(target_name)s",
      version     = "dev",
      description = "autogenerated thrift bindings for %(target_name)s",
      package_dir = { "": "gen-py" },
      packages    = %(packages)s,
      namespace_packages = %(namespace_packages)s)
"""
            boilerplate = boilerplate % {
                'target_name': self.target._create_id(),
                'genpy_root': genpy_root,
                'packages': repr(list(packages)),
                'namespace_packages': repr(list(namespace_packages))
            }

            self.chroot.write(boilerplate.encode('utf-8'),
                              os.path.join(self.codegen_root, 'setup.py'))

        dump_setup_py(self.detected_packages, self.detected_namespace_packages)

        egg_root = os.path.join(self.chroot.path(), self.codegen_root)
        egg_path = EggBuilder().build_egg(egg_root, self.target)
        return egg_path
Esempio n. 16
0
class PythonThriftBuilder(object):
    """
    Thrift builder.
  """
    class UnknownPlatformException(Exception):
        def __init__(self, platform):
            Exception.__init__(self, "Unknown platform: %s!" % str(platform))

    class CodeGenerationException(Exception):
        pass

    def __init__(self, target, root_dir):
        self.target = target
        self.root = root_dir
        distdir = os.path.join(self.root, 'dist')
        self.chroot = RelativeChroot(root_dir, distdir, target.name)
        codegen_root = tempfile.mkdtemp(dir=self.chroot.path(),
                                        prefix='codegen.')
        self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
        self.detected_packages = set()
        self.detected_namespace_packages = set()

    def __del__(self):
        self.cleanup()

    def packages(self):
        return self.detected_packages

    def cleanup(self):
        safe_rmtree(self.chroot.path())

    @staticmethod
    def thrift_binary(root_dir):
        # TODO(Brian Wickman):  Will we ever need to make this smarter?
        PLATMAP = {
            ('darwin', 'i386'): ['mac', '10.6', '0.5.0-finagle', 'thrift'],
            ('darwin', 'x86_64'): ['mac', '10.6', '0.5.0-finagle', 'thrift'],
            ('darwin', 'x86_64'): ['mac', '10.7', '0.5.0-finagle', 'thrift'],
            ('linux', 'x86_64'):
            ['linux', 'x86_64', '0.5.0-finagle', 'thrift'],
            ('linux', 'i686'): ['linux', 'i386', '0.5.0-finagle', 'thrift']
        }
        uname = os.uname()
        platform = (uname[0].lower(), uname[4])
        if platform not in PLATMAP:
            raise PythonThriftBuilder.UnknownPlatformException(platform)
        return os.path.join(root_dir, 'build-support', 'bin', 'thrift',
                            *PLATMAP[platform])

    def run_thrifts(self):
        for src in self.target.sources:
            if not self._run_thrift(src):
                raise PythonThriftBuilder.CodeGenerationException(
                    "Could not generate .py from %s!" % src)

    def _run_thrift(self, source):
        thrift_file = source
        thrift_abs_path = os.path.join(self.root, self.target.target_base,
                                       thrift_file)
        thrift_abs_path = os.path.abspath(thrift_abs_path)

        args = [
            PythonThriftBuilder.thrift_binary(self.root), '--gen',
            'py:new_style', '-o', self.codegen_root, thrift_abs_path
        ]

        cwd = os.getcwd()
        os.chdir(self.chroot.path())
        try:
            po = subprocess.Popen(args)
        finally:
            os.chdir(cwd)
        rv = po.wait()
        if rv != 0:
            comm = po.communicate()
            print('thrift generation failed!', file=sys.stderr)
            print('STDOUT', file=sys.stderr)
            print(comm[0], file=sys.stderr)
            print('STDERR', file=sys.stderr)
            print(comm[1], file=sys.stderr)
        return rv == 0

    @staticmethod
    def path_to_module(path):
        return path.replace(os.path.sep, '.')

    def build_egg(self):
        # autogenerate the python files that we bundle up
        self.run_thrifts()

        genpy_root = os.path.join(self.chroot.path(), self.codegen_root,
                                  'gen-py')
        for dir, _, files in os.walk(os.path.normpath(genpy_root)):
            reldir = os.path.relpath(dir, genpy_root)
            if reldir == '.': continue
            if '__init__.py' not in files: continue
            init_py_abspath = os.path.join(dir, '__init__.py')
            module_path = self.path_to_module(reldir)
            self.detected_packages.add(module_path)
            # A namespace package is one that is just a container for other
            # modules and subpackages. Setting their __init__.py files as follows
            # allows them to be distributed across multiple eggs. Without this you
            # couldn't have this egg share any package prefix with any other module
            # in any other egg or in the source tree.
            #
            # Note that the thrift compiler should always generate empty __init__.py
            # files, but we test for this anyway, just in case that changes.
            if len(files) == 1 and os.path.getsize(init_py_abspath) == 0:
                with open(init_py_abspath, 'wb') as f:
                    f.write(
                        b"__import__('pkg_resources').declare_namespace(__name__)"
                    )
                self.detected_namespace_packages.add(module_path)

        if not self.detected_packages:
            raise PythonThriftBuilder.CodeGenerationException(
                'No Thrift structures declared in %s!' % self.target)

        def dump_setup_py(packages, namespace_packages):
            boilerplate = """
from setuptools import setup

setup(name        = "%(target_name)s",
      version     = "dev",
      description = "autogenerated thrift bindings for %(target_name)s",
      package_dir = { "": "gen-py" },
      packages    = %(packages)s,
      namespace_packages = %(namespace_packages)s)
"""
            boilerplate = boilerplate % {
                'target_name': self.target._create_id(),
                'genpy_root': genpy_root,
                'packages': repr(list(packages)),
                'namespace_packages': repr(list(namespace_packages))
            }

            self.chroot.write(boilerplate.encode('utf-8'),
                              os.path.join(self.codegen_root, 'setup.py'))

        dump_setup_py(self.detected_packages, self.detected_namespace_packages)

        egg_root = os.path.join(self.chroot.path(), self.codegen_root)
        egg_path = EggBuilder().build_egg(egg_root, self.target)
        return egg_path
Esempio n. 17
0
class PythonThriftBuilder(object):
  """
    Thrift builder.
  """
  class UnknownPlatformException(Exception):
    def __init__(self, platform):
      Exception.__init__(self, "Unknown platform: %s!" % str(platform))
  class CodeGenerationException(Exception): pass

  def __init__(self, target, root_dir, config):
    self.target = target
    self.root = root_dir
    self.config = config
    distdir = os.path.join(self.root, 'dist')
    self.chroot = RelativeChroot(root_dir, distdir, target.name)
    codegen_root = tempfile.mkdtemp(dir=self.chroot.path(), prefix='codegen.')
    self.codegen_root = os.path.relpath(codegen_root, self.chroot.path())
    self.detected_packages = set()
    self.detected_namespace_packages = set()

  def __del__(self):
    self.cleanup()

  def packages(self):
    return self.detected_packages

  def cleanup(self):
    safe_rmtree(self.chroot.path())

  def run_thrifts(self):
    def is_py_thrift(target):
      return isinstance(target, PythonThriftLibrary)
    bases, roots = calculate_compile_roots([self.target], is_py_thrift)

    for src in roots:
      if not self._run_thrift(src, bases):
        raise PythonThriftBuilder.CodeGenerationException(
          "Could not generate .py from %s!" % src)

  def _run_thrift(self, source, bases):
    thrift_file = source
    thrift_abs_path = os.path.join(self.root, thrift_file)
    thrift_abs_path = os.path.abspath(thrift_abs_path)

    args = [
      select_thrift_binary(self.config),
      '--gen',
      'py:new_style,utf8strings',
      '-recurse',
      '-o',
      os.path.join(self.chroot.path(), self.codegen_root)
    ]
    for base in bases:
      args.extend(('-I', base))
    args.append(thrift_abs_path)

    po = subprocess.Popen(args)
    rv = po.wait()
    if rv != 0:
      comm = po.communicate()
      print('thrift generation failed!', file=sys.stderr)
      print('STDOUT', file=sys.stderr)
      print(comm[0], file=sys.stderr)
      print('STDERR', file=sys.stderr)
      print(comm[1], file=sys.stderr)
    return rv == 0

  @staticmethod
  def path_to_module(path):
    return path.replace(os.path.sep, '.')

  def build_egg(self):
    # autogenerate the python files that we bundle up
    self.run_thrifts()

    genpy_root = os.path.join(self.chroot.path(), self.codegen_root, 'gen-py')
    for dir, _, files in os.walk(os.path.normpath(genpy_root)):
      reldir = os.path.relpath(dir, genpy_root)
      if reldir == '.': continue
      if '__init__.py' not in files: continue
      init_py_abspath = os.path.join(dir, '__init__.py')
      module_path = self.path_to_module(reldir)
      self.detected_packages.add(module_path)
      # A namespace package is one that is just a container for other
      # modules and subpackages. Setting their __init__.py files as follows
      # allows them to be distributed across multiple eggs. Without this you
      # couldn't have this egg share any package prefix with any other module
      # in any other egg or in the source tree.
      #
      # Note that the thrift compiler should always generate empty __init__.py
      # files, but we test for this anyway, just in case that changes.
      if len(files) == 1 and os.path.getsize(init_py_abspath) == 0:
        with open(init_py_abspath, 'wb') as f:
          f.write(b"__import__('pkg_resources').declare_namespace(__name__)")
        self.detected_namespace_packages.add(module_path)

    if not self.detected_packages:
      raise PythonThriftBuilder.CodeGenerationException(
        'No Thrift structures declared in %s!' % self.target)

    def dump_setup_py(packages, namespace_packages):
      boilerplate = """
from setuptools import setup

setup(name        = "%(target_name)s",
      version     = "dev",
      description = "autogenerated thrift bindings for %(target_name)s",
      package_dir = { "": "gen-py" },
      packages    = %(packages)s,
      namespace_packages = %(namespace_packages)s)
"""
      boilerplate = boilerplate % {
        'target_name': self.target._create_id(),
        'genpy_root': genpy_root,
        'packages': repr(list(packages)),
        'namespace_packages': repr(list(namespace_packages))
      }

      self.chroot.write(boilerplate.encode('utf-8'), os.path.join(self.codegen_root, 'setup.py'))
    dump_setup_py(self.detected_packages, self.detected_namespace_packages)

    egg_root = os.path.join(self.chroot.path(), self.codegen_root)
    egg_path = EggBuilder().build_egg(egg_root, self.target)
    return egg_path