Beispiel #1
0
 def __init__(self, target, root_dir):
   self._target = target
   self._root = root_dir
   self._cache = BuildCache(os.path.join(root_dir, '.pants.d', 'py_artifact_cache'))
   distdir = os.path.join(root_dir, 'dist')
   distpath = tempfile.mktemp(dir=distdir, prefix=target.name)
   self.env = PythonEnvironment(distpath)
Beispiel #2
0
    def __init__(self, target, root_dir, extra_targets=None):
        self._config = Config.load()
        self._target = target
        self._root = root_dir
        self._cache = BuildCache(
            os.path.join(self._config.get("python-setup", "artifact_cache"), "%s" % PythonIdentity.get())
        )
        self._extra_targets = list(extra_targets) if extra_targets is not None else []
        self._extra_targets.append(self._get_common_python())

        cachedir = self._config.get("python-setup", "cache")
        safe_mkdir(cachedir)
        self._eggcache = cachedir

        local_repo = "file://%s" % os.path.realpath(cachedir)
        self._repos = [local_repo] + self._config.getlist("python-setup", "repos")
        self._fetcher = ReqFetcher(repos=self._repos, cache=cachedir)
        self._index = None
        for index in self._config.getlist("python-setup", "indices"):
            if PythonChroot.can_contact_index(index):
                self._index = index
                break
        self._additional_reqs = set()

        distdir = self._config.getdefault("pants_distdir")
        distpath = tempfile.mktemp(dir=distdir, prefix=target.name)
        self.env = PythonEnvironment(distpath)
Beispiel #3
0
class PythonChroot(object):
  class InvalidDependencyException(Exception):
    def __init__(self, target):
      Exception.__init__(self, "Not a valid Python dependency! Found: %s" % target)

  class BuildFailureException(Exception):
    def __init__(self, target):
      Exception.__init__(self, "Not a valid Python dependency! Found: %s" % target)

  def __init__(self, target, root_dir):
    self._target = target
    self._root = root_dir
    self._cache = BuildCache(os.path.join(root_dir, '.pants.d', 'py_artifact_cache'))
    distdir = os.path.join(root_dir, 'dist')
    distpath = tempfile.mktemp(dir=distdir, prefix=target.name)
    self.env = PythonEnvironment(distpath)

  def __del__(self):
    if os.getenv('PANTS_LEAVE_CHROOT') is None:
      try:
        shutil.rmtree(self.path())
      except OSError as e:
        if e.errno != errno.ENOENT:
          raise

  def path(self):
    return self.env.path()

  def _dump_library(self, library):
    def translate_module(module):
      if module is None:
        module = ''
      return module.replace('.', os.path.sep)

    def copy_to_chroot(base, path, relative_to, add_function):
      src = os.path.join(self._root, base, path)
      dst = os.path.join(translate_module(relative_to), path)
      add_function(src, dst)

    template = library._create_template_data()
    print '  Dumping library: %s [relative module: %s]' % (library, template.module)
    for filename in template.sources:
      copy_to_chroot(template.template_base, filename, template.module, self.env.add_source)
    for filename in template.resources:
      copy_to_chroot(template.template_base, filename, template.module, self.env.add_resource)

  def _dump_egg(self, egg):
    target_name = os.path.pathsep.join(sorted(os.path.basename(e) for e in egg.eggs))
    cache_key = self._cache.key_for(target_name, egg.eggs)
    if self._cache.needs_update(cache_key):
      print '  Dumping egg: %s' % egg
      prefixes, all_added_files = set(), set()
      for egg_path in egg.eggs:
        egg_dep = PythonDependency.from_eggs(egg_path)
        prefix, added_files = self.env.add_dependency(egg_dep)
        all_added_files.update(added_files)
        prefixes.add(prefix)
      assert len(prefixes) == 1, 'Ambiguous egg environment!'
      self._cache.update(cache_key, all_added_files, artifact_root=prefixes.pop())
    else:
      print '  Dumping (cached) egg: %s' % egg
      self._cache.use_cached_files(cache_key, self.env.add_dependency_file)

  def _dump_setuptools(self):
    SETUPTOOLS = 'distribute-0.6.21-py2.6.egg'
    print '  Dumping setuptools: %s' % SETUPTOOLS
    data = pkgutil.get_data(__name__, os.path.join('bootstrap', SETUPTOOLS))
    with temporary_dir() as tempdir:
      egg_path = os.path.join(tempdir, SETUPTOOLS)
      with open(egg_path, 'w') as fp:
        fp.write(data)
      egg_dep = PythonDependency.from_eggs(egg_path)
      self.env.add_dependency(egg_dep)

  # TODO(wickman) Just add write() to self.env and do this with pkg_resources or
  # just build an egg for twitter.common.python.
  def _get_common_python(self):
    return Target.get(Address.parse(self._root, 'src/python/twitter/common/python'))

  def _dump_bin(self, binary_name, base):
    src = os.path.join(self._root, base, binary_name)
    print '  Dumping binary: %s' % binary_name
    self.env.set_executable(src, os.path.basename(src))

  def _dump_thrift_library(self, library):
    self._dump_built_library(library, PythonThriftBuilder(library, self._root))

  def _dump_antlr_library(self, library):
    self._dump_built_library(library, PythonAntlrBuilder(library, self._root))

  def _dump_built_library(self, library, builder):
    absolute_sources = library.expand_files()
    absolute_sources.sort()
    cache_key = self._cache.key_for(library._create_id(), absolute_sources)
    if not self._cache.needs_update(cache_key):
      print '  Generating (cached) %s...' % library
      self._cache.use_cached_files(cache_key, self.env.add_dependency_file)
    else:
      print '  Generating %s...' % library
      egg_file = builder.build_egg()

      if egg_file:
        src_egg_file = egg_file
        dst_egg_file = os.path.join(os.path.dirname(egg_file),
            cache_key.hash + '_' + os.path.basename(egg_file))
        os.rename(src_egg_file, dst_egg_file)
        self._cache.update(cache_key, [dst_egg_file])
        egg_dep = PythonDependency.from_eggs(dst_egg_file)

        for pkg in builder.packages():
          print '    found namespace: %s' % pkg
        print '    copying...',
        self.env.add_dependency(egg_dep)
        print 'done.'
      else:
        print '   Failed!'
        raise PythonChroot.BuildFailureException("Failed to build %s!" % library)

  def build_dep_tree(self, input_target):
    target = copy.deepcopy(input_target)
    common_python = self._get_common_python()
    if common_python not in target.dependencies:
      target.dependencies.add(common_python)

    libraries = set()
    eggs = set()
    binaries = set()
    thrifts = set()
    antlrs = set()

    def add_dep(trg):
      if isinstance(trg, PythonLibrary):
        if trg.sources:
          libraries.add(trg)
        for egg in [dep for dep in trg.dependencies if isinstance(dep, PythonEgg)]:
          eggs.add(egg)
      elif isinstance(trg, PythonEgg):
        eggs.add(trg)
      elif isinstance(trg, PythonBinary):
        binaries.add(trg)
      elif isinstance(trg, PythonThriftLibrary):
        thrifts.add(trg)
      elif isinstance(trg, PythonAntlrLibrary):
        antlrs.add(trg)
      elif isinstance(trg, PythonTests):
        # do not dump test sources/resources, but dump their
        # dependencies.
        pass
      else:
        raise PythonChroot.InvalidDependencyException(trg)
      return [dep for dep in trg.dependencies if not isinstance(dep, PythonEgg)]

    target.walk(lambda t: add_dep(t), lambda typ: not isinstance(typ, PythonEgg))

    return libraries, eggs, binaries, thrifts, antlrs

  def dump(self):
    print 'Building PythonBinary %s:' % self._target
    libraries, eggs, binaries, thrifts, antlrs = self.build_dep_tree(self._target)

    for lib in libraries:
      self._dump_library(lib)
    self._dump_setuptools()
    for egg in eggs:
      self._dump_egg(egg)
    for thr in thrifts:
      self._dump_thrift_library(thr)
    for antlr in antlrs:
      self._dump_antlr_library(antlr)
    if len(binaries) > 1:
      print >> sys.stderr, 'WARNING: Target has multiple python_binary targets!'
    for binary in binaries:
      self._dump_bin(binary.sources[0], binary.target_base)
    self.env.freeze()
    return self.env
Beispiel #4
0
class PythonChroot(object):
    class InvalidDependencyException(Exception):
        def __init__(self, target):
            Exception.__init__(self, "Not a valid Python dependency! Found: %s" % target)

    class BuildFailureException(Exception):
        def __init__(self, target):
            Exception.__init__(self, "Not a valid Python dependency! Found: %s" % target)

    @staticmethod
    def can_contact_index(url, timeout=Amount(5, Time.SECONDS)):
        try:
            response = urlopen(url, timeout=timeout.as_(Time.SECONDS))
            return response.code == 200
        except URLError:
            return False

    def __init__(self, target, root_dir, extra_targets=None):
        self._config = Config.load()
        self._target = target
        self._root = root_dir
        self._cache = BuildCache(
            os.path.join(self._config.get("python-setup", "artifact_cache"), "%s" % PythonIdentity.get())
        )
        self._extra_targets = list(extra_targets) if extra_targets is not None else []
        self._extra_targets.append(self._get_common_python())

        cachedir = self._config.get("python-setup", "cache")
        safe_mkdir(cachedir)
        self._eggcache = cachedir

        local_repo = "file://%s" % os.path.realpath(cachedir)
        self._repos = [local_repo] + self._config.getlist("python-setup", "repos")
        self._fetcher = ReqFetcher(repos=self._repos, cache=cachedir)
        self._index = None
        for index in self._config.getlist("python-setup", "indices"):
            if PythonChroot.can_contact_index(index):
                self._index = index
                break
        self._additional_reqs = set()

        distdir = self._config.getdefault("pants_distdir")
        distpath = tempfile.mktemp(dir=distdir, prefix=target.name)
        self.env = PythonEnvironment(distpath)

    def __del__(self):
        if os.getenv("PANTS_LEAVE_CHROOT") is None:
            try:
                shutil.rmtree(self.path())
            except OSError as e:
                if e.errno != errno.ENOENT:
                    raise

    def debug(self, msg, indent=0):
        if os.getenv("PANTS_VERBOSE") is not None:
            print("%s%s" % (" " * indent, msg))

    def path(self):
        return self.env.path()

    def _dump_library(self, library):
        def translate_module(module):
            if module is None:
                module = ""
            return module.replace(".", os.path.sep)

        def copy_to_chroot(base, path, relative_to, add_function):
            src = os.path.join(self._root, base, path)
            dst = os.path.join(translate_module(relative_to), path)
            add_function(src, dst)

        self.debug("  Dumping library: %s [relative module: %s]" % (library, library.module))
        for filename in library.sources:
            copy_to_chroot(library.target_base, filename, library.module, self.env.add_source)
        for filename in library.resources:
            copy_to_chroot(library.target_base, filename, library.module, self.env.add_resource)

    def _dump_egg(self, egg):
        if isinstance(egg, PythonEgg):
            target_name = os.path.pathsep.join(sorted(os.path.basename(e) for e in egg.eggs))
            cache_key = self._cache.key_for(target_name, egg.eggs)
            if self._cache.needs_update(cache_key):
                self.debug("  Dumping egg: %s" % egg)
                prefixes, all_added_files = set(), set()
                for egg_path in egg.eggs:
                    egg_dep = PythonDependency.from_eggs(egg_path)
                    prefix, added_files = self.env.add_dependency(egg_dep)
                    all_added_files.update(added_files)
                    prefixes.add(prefix)
                assert len(prefixes) == 1, "Ambiguous egg environment!"
                self._cache.update(cache_key, all_added_files, artifact_root=prefixes.pop())
            else:
                self.debug("  Dumping (cached) egg: %s" % egg)
                self._cache.use_cached_files(cache_key, self.env.add_dependency_file)
        elif isinstance(egg, PythonDependency):
            self.debug("  Dumping PythonDependency: %s" % egg)
            self.env.add_dependency(egg)
        else:
            raise PythonChroot.InvalidDependencyException("Unknown egg dependency %s" % egg)

    def _dump_distribution(self, dist):
        self.debug("  Dumping distribution: .../%s" % os.path.basename(dist.location))
        egg_dep = PythonDependency.from_distributions(dist)
        self.env.add_dependency(egg_dep)

    # TODO(wickman) Just add write() to self.env and do this with pkg_resources or
    # just build an egg for twitter.common.python.
    def _get_common_python(self):
        return Target.get(Address.parse(self._root, "src/python/twitter/common/python"))

    def _dump_bin(self, binary_name, base):
        src = os.path.join(self._root, base, binary_name)
        self.debug("  Dumping binary: %s" % binary_name)
        self.env.set_executable(src, os.path.basename(src))

    def _dump_thrift_library(self, library):
        self._dump_built_library(library, PythonThriftBuilder(library, self._root))

    def _dump_antlr_library(self, library):
        self._dump_built_library(library, PythonAntlrBuilder(library, self._root))

    def _dump_built_library(self, library, builder):
        absolute_sources = library.expand_files()
        absolute_sources.sort()
        cache_key = self._cache.key_for(library._create_id(), absolute_sources)
        if not self._cache.needs_update(cache_key):
            self.debug("  Generating (cached) %s..." % library)
            self._cache.use_cached_files(cache_key, self.env.add_dependency_file)
        else:
            self.debug("  Generating %s..." % library)
            egg_file = builder.build_egg()

            if egg_file:
                src_egg_file = egg_file
                dst_egg_file = os.path.join(
                    os.path.dirname(egg_file), cache_key.hash + "_" + os.path.basename(egg_file)
                )
                os.rename(src_egg_file, dst_egg_file)
                self._cache.update(cache_key, [dst_egg_file])
                egg_dep = PythonDependency.from_eggs(dst_egg_file)

                for pkg in builder.packages():
                    self.debug("    found namespace: %s" % pkg)
                self.debug("    copying...")
                self.env.add_dependency(egg_dep)
                self.debug("done.")
            else:
                self.debug("   Failed!")
                raise PythonChroot.BuildFailureException("Failed to build %s!" % library)

    def aggregate_targets(self, targets):
        libraries, eggs, reqs, binaries, thrifts, antlrs = set(), set(), set(), set(), set(), set()
        for target in targets:
            (addl_libraries, addl_eggs, addl_reqs, addl_binaries, addl_thrifts, addl_antlrs) = self.build_dep_tree(
                target
            )

            libraries.update(addl_libraries)
            eggs.update(addl_eggs)
            reqs.update(addl_reqs)
            binaries.update(addl_binaries)
            thrifts.update(addl_thrifts)
            antlrs.update(addl_antlrs)

        return libraries, eggs, reqs, binaries, thrifts, antlrs

    def build_dep_tree(self, input_target):
        target = copy.deepcopy(input_target)
        common_python = self._get_common_python()
        if common_python not in target.dependencies:
            target.dependencies.add(common_python)

        libraries, eggs, reqs, binaries, thrifts, antlrs = set(), set(), set(), set(), set(), set()

        def add_dep(trg):
            if isinstance(trg, PythonLibrary):
                if trg.sources or trg.resources:
                    libraries.add(trg)
                for dep in trg.dependencies:
                    if isinstance(dep, PythonEgg):
                        eggs.add(dep)
            elif isinstance(trg, PythonEgg):
                eggs.add(trg)
            elif isinstance(trg, PythonRequirement):
                reqs.add(trg)
            elif isinstance(trg, PythonBinary):
                binaries.add(trg)
            elif isinstance(trg, PythonThriftLibrary):
                thrifts.add(trg)
            elif isinstance(trg, PythonAntlrLibrary):
                antlrs.add(trg)
            elif isinstance(trg, PythonTests):
                pass
            else:
                raise PythonChroot.InvalidDependencyException(trg)
            return trg.dependencies if hasattr(trg, "dependencies") else []

        target.walk(lambda t: add_dep(t))

        return libraries, eggs, reqs, binaries, thrifts, antlrs

    def cache_egg_if_necessary(self, egg):
        target_file = os.path.join(self._eggcache, os.path.basename(egg))
        if not os.path.exists(target_file):
            if os.path.isdir(egg):
                shutil.copytree(egg, target_file)
            else:
                shutil.copyfile(egg, target_file)

    def _dump_req(self, req):
        self.debug("    - Fetching requirement: %s" % req)
        fetched_egg = self._fetcher.fetch(req)
        if fetched_egg is None:
            print("ERROR: Could not find %s!" % req, file=sys.stderr)
        else:
            if fetched_egg.endswith(".egg"):
                self._dump_egg(PythonDependency.from_eggs(fetched_egg))
                self.cache_egg_if_necessary(fetched_egg)
            else:
                if self._index is None:
                    self.debug("ERROR: Could not contact any indices!  Your application may not " "work properly.")
                    return
                self.debug("      => Building %s" % fetched_egg)
                distributions = ReqBuilder.install_requirement(fetched_egg, index=self._index, repositories=self._repos)
                if not distributions:
                    print("WARNING: Unable to build a working distribution from %s!" % fetched_egg, file=sys.stderr)
                    print("Your application may not work properly.", file=sys.stderr)
                else:
                    for distribution in distributions:
                        if distribution.location.endswith(".egg"):
                            self.cache_egg_if_necessary(distribution.location)
                        self._dump_distribution(distribution)

    def _dump_python_req(self, python_req):
        self.debug("  Dumping Requirement(%s)" % python_req.name)
        self._dump_req(python_req._requirement)

    def add_req(self, req):
        assert isinstance(req, pkg_resources.Requirement)
        self._additional_reqs.add(req)

    def dump(self):
        self.debug("Building PythonBinary %s:" % self._target)

        libraries, eggs, reqs, binaries, thrifts, antlrs = self.aggregate_targets([self._target] + self._extra_targets)

        bare_reqs = set([pkg_resources.Requirement.parse("distribute")])
        bare_reqs.update(self._additional_reqs)

        for lib in libraries:
            self._dump_library(lib)
        for req in bare_reqs:
            self._dump_req(req)
        for egg in eggs:
            self._dump_egg(egg)
        for req in reqs:
            self._dump_python_req(req)
        for thr in thrifts:
            self._dump_thrift_library(thr)
        for antlr in antlrs:
            self._dump_antlr_library(antlr)
        if len(binaries) > 1:
            print("WARNING: Target has multiple python_binary targets!", file=sys.stderr)
        for binary in binaries:
            self._dump_bin(binary.sources[0], binary.target_base)
        self.env.freeze()
        return self.env