Beispiel #1
0
    def __init__(self,
                 python_setup,
                 python_repos,
                 ivy_bootstrapper,
                 thrift_binary_factory,
                 interpreter,
                 builder,
                 targets,
                 platforms,
                 extra_requirements=None,
                 log=None):
        self._python_setup = python_setup
        self._python_repos = python_repos
        self._ivy_bootstrapper = ivy_bootstrapper
        self._thrift_binary_factory = thrift_binary_factory

        self._interpreter = interpreter
        self._builder = builder
        self._targets = targets
        self._platforms = platforms
        self._extra_requirements = list(
            extra_requirements) if extra_requirements else []
        self._logger = log or logger

        # Note: unrelated to the general pants artifact cache.
        self._artifact_cache_root = os.path.join(
            self._python_setup.artifact_cache_dir,
            str(self._interpreter.identity))
        self._key_generator = CacheKeyGenerator()
        self._build_invalidator = BuildInvalidator(self._artifact_cache_root)
Beispiel #2
0
    def __init__(self,
                 results_dir_root,
                 cache_key_generator,
                 build_invalidator_dir,
                 invalidate_dependents,
                 fingerprint_strategy=None,
                 invalidation_report=None,
                 task_name=None,
                 task_version=None,
                 artifact_write_callback=lambda _: None):
        """
    :API: public
    """
        self._cache_key_generator = cache_key_generator
        self._task_name = task_name or 'UNKNOWN'
        self._task_version = task_version or 'Unknown_0'
        self._invalidate_dependents = invalidate_dependents
        self._invalidator = BuildInvalidator(build_invalidator_dir)
        self._fingerprint_strategy = fingerprint_strategy
        self._artifact_write_callback = artifact_write_callback
        self.invalidation_report = invalidation_report

        # Create the task-versioned prefix of the results dir, and a stable symlink to it
        # (useful when debugging).
        self._results_dir_prefix = os.path.join(
            results_dir_root,
            sha1(self._task_version).hexdigest()[:12])
        safe_mkdir(self._results_dir_prefix)
        stable_prefix = os.path.join(results_dir_root, self._STABLE_DIR_NAME)
        safe_delete(stable_prefix)
        relative_symlink(self._results_dir_prefix, stable_prefix)
Beispiel #3
0
 def __init__(self,
              cache_key_generator,
              build_invalidator_dir,
              invalidate_dependents,
              fingerprint_strategy=None,
              invalidation_report=None,
              task_name=None):
   self._cache_key_generator = cache_key_generator
   self._task_name = task_name or 'UNKNOWN'
   self._invalidate_dependents = invalidate_dependents
   self._invalidator = BuildInvalidator(build_invalidator_dir)
   self._fingerprint_strategy = fingerprint_strategy
   self.invalidation_report = invalidation_report
Beispiel #4
0
  def __init__(self,
               python_setup,
               python_repos,
               ivy_bootstrapper,
               thrift_binary_factory,
               interpreter,
               builder,
               targets,
               platforms,
               extra_requirements=None,
               log=None):
    self._python_setup = python_setup
    self._python_repos = python_repos
    self._ivy_bootstrapper = ivy_bootstrapper
    self._thrift_binary_factory = thrift_binary_factory

    self._interpreter = interpreter
    self._builder = builder
    self._targets = targets
    self._platforms = platforms
    self._extra_requirements = list(extra_requirements) if extra_requirements else []
    self._logger = log or logger

    # Note: unrelated to the general pants artifact cache.
    self._artifact_cache_root = os.path.join(
      self._python_setup.artifact_cache_dir, str(self._interpreter.identity))
    self._key_generator = CacheKeyGenerator()
    self._build_invalidator = BuildInvalidator(self._artifact_cache_root)
Beispiel #5
0
  def __init__(self,
               results_dir_root,
               cache_key_generator,
               build_invalidator_dir,
               invalidate_dependents,
               fingerprint_strategy=None,
               invalidation_report=None,
               task_name=None,
               task_version=None,
               artifact_write_callback=lambda _: None):
    """
    :API: public
    """
    self._cache_key_generator = cache_key_generator
    self._task_name = task_name or 'UNKNOWN'
    self._task_version = task_version or 'Unknown_0'
    self._invalidate_dependents = invalidate_dependents
    self._invalidator = BuildInvalidator(build_invalidator_dir)
    self._fingerprint_strategy = fingerprint_strategy
    self._artifact_write_callback = artifact_write_callback
    self.invalidation_report = invalidation_report

    # Create the task-versioned prefix of the results dir, and a stable symlink to it (useful when debugging).
    self._results_dir_prefix = os.path.join(results_dir_root, sha1(self._task_version).hexdigest()[:12])
    safe_mkdir(self._results_dir_prefix)
    stable_prefix = os.path.join(results_dir_root, self._STABLE_DIR_NAME)
    safe_delete(stable_prefix)
    relative_symlink(self._results_dir_prefix, stable_prefix)
Beispiel #6
0
 def setUp(self):
   super(InvalidationCacheManagerTest, self).setUp()
   self._dir = tempfile.mkdtemp()
   self.cache_manager = InvalidationCacheManager(
     results_dir_root=os.path.join(self._dir, 'results'),
     cache_key_generator=CacheKeyGenerator(),
     build_invalidator=BuildInvalidator(os.path.join(self._dir, 'build_invalidator')),
     invalidate_dependents=True,
   )
Beispiel #7
0
 def setUp(self):
     super().setUp()
     self._dir = tempfile.mkdtemp()
     self.cache_manager = InvalidationCacheManager(
         results_dir_root=os.path.join(self._dir, "results"),
         cache_key_generator=CacheKeyGenerator(),
         build_invalidator=BuildInvalidator(
             os.path.join(self._dir, "build_invalidator")),
         invalidate_dependents=True,
         task_version_slug="deadbeef",
     )
Beispiel #8
0
 def __init__(
     self,
     cache_key_generator,
     build_invalidator_dir,
     invalidate_dependents,
     fingerprint_strategy=None,
     invalidation_report=None,
     task_name=None,
     task_version=None,
 ):
     """
 :API: public
 """
     self._cache_key_generator = cache_key_generator
     self._task_name = task_name or 'UNKNOWN'
     self._task_version = task_version or 'Unknown_0'
     self._invalidate_dependents = invalidate_dependents
     self._invalidator = BuildInvalidator(build_invalidator_dir)
     self._fingerprint_strategy = fingerprint_strategy
     self.invalidation_report = invalidation_report
Beispiel #9
0
 def __init__(self,
              cache_key_generator,
              build_invalidator_dir,
              invalidate_dependents,
              fingerprint_strategy=None,
              invalidation_report=None,
              task_name=None,
              task_version=None,):
   """
   :API: public
   """
   self._cache_key_generator = cache_key_generator
   self._task_name = task_name or 'UNKNOWN'
   self._task_version = task_version or 'Unknown_0'
   self._invalidate_dependents = invalidate_dependents
   self._invalidator = BuildInvalidator(build_invalidator_dir)
   self._fingerprint_strategy = fingerprint_strategy
   self.invalidation_report = invalidation_report
Beispiel #10
0
 def invalidate(self):
     """Invalidates all targets for this task."""
     BuildInvalidator(self._build_invalidator_dir).force_invalidate_all()
Beispiel #11
0
class InvalidationCacheManager(object):
  """Manages cache checks, updates and invalidation keeping track of basic change
  and invalidation statistics.
  Note that this is distinct from the ArtifactCache concept, and should probably be renamed.
  """

  class CacheValidationError(Exception):
    """Indicates a problem accessing the cache."""

  _STABLE_DIR_NAME = 'current'

  def __init__(self,
               results_dir_root,
               cache_key_generator,
               build_invalidator_dir,
               invalidate_dependents,
               fingerprint_strategy=None,
               invalidation_report=None,
               task_name=None,
               task_version=None,
               artifact_write_callback=lambda _: None):
    """
    :API: public
    """
    self._cache_key_generator = cache_key_generator
    self._task_name = task_name or 'UNKNOWN'
    self._task_version = task_version or 'Unknown_0'
    self._invalidate_dependents = invalidate_dependents
    self._invalidator = BuildInvalidator(build_invalidator_dir)
    self._fingerprint_strategy = fingerprint_strategy
    self._artifact_write_callback = artifact_write_callback
    self.invalidation_report = invalidation_report

    # Create the task-versioned prefix of the results dir, and a stable symlink to it (useful when debugging).
    self._results_dir_prefix = os.path.join(results_dir_root, sha1(self._task_version).hexdigest()[:12])
    safe_mkdir(self._results_dir_prefix)
    stable_prefix = os.path.join(results_dir_root, self._STABLE_DIR_NAME)
    safe_delete(stable_prefix)
    relative_symlink(self._results_dir_prefix, stable_prefix)

  def update(self, vts):
    """Mark a changed or invalidated VersionedTargetSet as successfully processed."""
    for vt in vts.versioned_targets:
      vt.ensure_legal()
      if not vt.valid:
        self._invalidator.update(vt.cache_key)
        vt.valid = True
        self._artifact_write_callback(vt)
    if not vts.valid:
      vts.ensure_legal()
      self._invalidator.update(vts.cache_key)
      vts.valid = True
      self._artifact_write_callback(vts)

  def force_invalidate(self, vts):
    """Force invalidation of a VersionedTargetSet."""
    for vt in vts.versioned_targets:
      self._invalidator.force_invalidate(vt.cache_key)
      vt.valid = False
    self._invalidator.force_invalidate(vts.cache_key)
    vts.valid = False

  def check(self,
            targets,
            topological_order=False):
    """Checks whether each of the targets has changed and invalidates it if so.

    Returns a list of VersionedTargetSet objects (either valid or invalid). The returned sets
    'cover' the input targets, with one caveat: if the FingerprintStrategy
    opted out of fingerprinting a target because it doesn't contribute to invalidation, then that
    target will be excluded from all_vts and invalid_vts.

    Callers can inspect these vts and rebuild the invalid ones, for example.
    """
    all_vts = self.wrap_targets(targets, topological_order=topological_order)
    invalid_vts = filter(lambda vt: not vt.valid, all_vts)
    return InvalidationCheck(all_vts, invalid_vts)

  @property
  def task_name(self):
    return self._task_name

  def results_dir_path(self, key, stable):
    """Return a results directory path for the given key.

    :param key: A CacheKey to generate an id for.
    :param stable: True to use a stable subdirectory, false to use a portion of the cache key to
      generate a path unique to the key.
    """
    # TODO: Shorten cache_key hashes in general?
    return os.path.join(
      self._results_dir_prefix,
      key.id,
      self._STABLE_DIR_NAME if stable else sha1(key.hash).hexdigest()[:12]
    )

  def wrap_targets(self, targets, topological_order=False):
    """Wrap targets and their computed cache keys in VersionedTargets.

    If the FingerprintStrategy opted out of providing a fingerprint for a target, that target will not
    have an associated VersionedTarget returned.

    Returns a list of VersionedTargets, each representing one input target.
    """
    def vt_iter():
      if topological_order:
        target_set = set(targets)
        sorted_targets = [t for t in reversed(sort_targets(targets)) if t in target_set]
      else:
        sorted_targets = sorted(targets)
      for target in sorted_targets:
        target_key = self._key_for(target)
        if target_key is not None:
          yield VersionedTarget(self, target, target_key)
    return list(vt_iter())

  def previous_key(self, cache_key):
    return self._invalidator.previous_key(cache_key)

  def _key_for(self, target):
    try:
      return self._cache_key_generator.key_for_target(target,
                                                      transitive=self._invalidate_dependents,
                                                      fingerprint_strategy=self._fingerprint_strategy)
    except Exception as e:
      # This is a catch-all for problems we haven't caught up with and given a better diagnostic.
      # TODO(Eric Ayers): If you see this exception, add a fix to catch the problem earlier.
      exc_info = sys.exc_info()
      new_exception = self.CacheValidationError("Problem validating target {} in {}: {}"
                                                .format(target.id, target.address.spec_path, e))

      raise self.CacheValidationError, new_exception, exc_info[2]
def test_env(content=TEST_CONTENT):
    with temporary_dir() as d:
        with tempfile.NamedTemporaryFile() as f:
            f.write(content)
            f.flush()
            yield f, CacheKeyGenerator(), BuildInvalidator(d)
Beispiel #13
0
class InvalidationCacheManager(object):
    """Manages cache checks, updates and invalidation keeping track of basic change
  and invalidation statistics.
  Note that this is distinct from the ArtifactCache concept, and should probably be renamed.
  """
    class CacheValidationError(Exception):
        """Indicates a problem accessing the cache."""

    _STABLE_DIR_NAME = 'current'

    def __init__(self,
                 results_dir_root,
                 cache_key_generator,
                 build_invalidator_dir,
                 invalidate_dependents,
                 fingerprint_strategy=None,
                 invalidation_report=None,
                 task_name=None,
                 task_version=None,
                 artifact_write_callback=lambda _: None):
        """
    :API: public
    """
        self._cache_key_generator = cache_key_generator
        self._task_name = task_name or 'UNKNOWN'
        self._task_version = task_version or 'Unknown_0'
        self._invalidate_dependents = invalidate_dependents
        self._invalidator = BuildInvalidator(build_invalidator_dir)
        self._fingerprint_strategy = fingerprint_strategy
        self._artifact_write_callback = artifact_write_callback
        self.invalidation_report = invalidation_report

        # Create the task-versioned prefix of the results dir, and a stable symlink to it
        # (useful when debugging).
        self._results_dir_prefix = os.path.join(
            results_dir_root,
            sha1(self._task_version).hexdigest()[:12])
        safe_mkdir(self._results_dir_prefix)
        stable_prefix = os.path.join(results_dir_root, self._STABLE_DIR_NAME)
        safe_delete(stable_prefix)
        relative_symlink(self._results_dir_prefix, stable_prefix)

    def update(self, vts):
        """Mark a changed or invalidated VersionedTargetSet as successfully processed."""
        for vt in vts.versioned_targets:
            vt.ensure_legal()
            if not vt.valid:
                self._invalidator.update(vt.cache_key)
                vt.valid = True
                self._artifact_write_callback(vt)
        if not vts.valid:
            vts.ensure_legal()
            self._invalidator.update(vts.cache_key)
            vts.valid = True
            self._artifact_write_callback(vts)

    def force_invalidate(self, vts):
        """Force invalidation of a VersionedTargetSet."""
        for vt in vts.versioned_targets:
            self._invalidator.force_invalidate(vt.cache_key)
            vt.valid = False
        self._invalidator.force_invalidate(vts.cache_key)
        vts.valid = False

    def check(self, targets, topological_order=False):
        """Checks whether each of the targets has changed and invalidates it if so.

    Returns a list of VersionedTargetSet objects (either valid or invalid). The returned sets
    'cover' the input targets, with one caveat: if the FingerprintStrategy
    opted out of fingerprinting a target because it doesn't contribute to invalidation, then that
    target will be excluded from all_vts and invalid_vts.

    Callers can inspect these vts and rebuild the invalid ones, for example.
    """
        all_vts = self.wrap_targets(targets,
                                    topological_order=topological_order)
        invalid_vts = filter(lambda vt: not vt.valid, all_vts)
        return InvalidationCheck(all_vts, invalid_vts)

    @property
    def task_name(self):
        return self._task_name

    def results_dir_path(self, key, stable):
        """Return a results directory path for the given key.

    :param key: A CacheKey to generate an id for.
    :param stable: True to use a stable subdirectory, false to use a portion of the cache key to
      generate a path unique to the key.
    """
        # TODO: Shorten cache_key hashes in general?
        return os.path.join(
            self._results_dir_prefix, key.id, self._STABLE_DIR_NAME
            if stable else sha1(key.hash).hexdigest()[:12])

    def wrap_targets(self, targets, topological_order=False):
        """Wrap targets and their computed cache keys in VersionedTargets.

    If the FingerprintStrategy opted out of providing a fingerprint for a target, that target will not
    have an associated VersionedTarget returned.

    Returns a list of VersionedTargets, each representing one input target.
    """
        def vt_iter():
            if topological_order:
                target_set = set(targets)
                sorted_targets = [
                    t for t in reversed(sort_targets(targets))
                    if t in target_set
                ]
            else:
                sorted_targets = sorted(targets)
            for target in sorted_targets:
                target_key = self._key_for(target)
                if target_key is not None:
                    yield VersionedTarget(self, target, target_key)

        return list(vt_iter())

    def previous_key(self, cache_key):
        return self._invalidator.previous_key(cache_key)

    def _key_for(self, target):
        try:
            return self._cache_key_generator.key_for_target(
                target,
                transitive=self._invalidate_dependents,
                fingerprint_strategy=self._fingerprint_strategy)
        except Exception as e:
            # This is a catch-all for problems we haven't caught up with and given a better diagnostic.
            # TODO(Eric Ayers): If you see this exception, add a fix to catch the problem earlier.
            exc_info = sys.exc_info()
            new_exception = self.CacheValidationError(
                "Problem validating target {} in {}: {}".format(
                    target.id, target.address.spec_path, e))

            raise self.CacheValidationError, new_exception, exc_info[2]
Beispiel #14
0
class InvalidationCacheManager(object):
  """Manages cache checks, updates and invalidation keeping track of basic change
  and invalidation statistics.
  Note that this is distinct from the ArtifactCache concept, and should probably be renamed.
  """

  class CacheValidationError(Exception):
    """Indicates a problem accessing the cache."""

  def __init__(self,
               cache_key_generator,
               build_invalidator_dir,
               invalidate_dependents,
               fingerprint_strategy=None,
               invalidation_report=None,
               task_name=None):
    self._cache_key_generator = cache_key_generator
    self._task_name = task_name or 'UNKNOWN'
    self._invalidate_dependents = invalidate_dependents
    self._invalidator = BuildInvalidator(build_invalidator_dir)
    self._fingerprint_strategy = fingerprint_strategy
    self.invalidation_report = invalidation_report

  def update(self, vts):
    """Mark a changed or invalidated VersionedTargetSet as successfully processed."""
    for vt in vts.versioned_targets:
      self._invalidator.update(vt.cache_key)
      vt.valid = True
    self._invalidator.update(vts.cache_key)
    vts.valid = True

  def force_invalidate(self, vts):
    """Force invalidation of a VersionedTargetSet."""
    for vt in vts.versioned_targets:
      self._invalidator.force_invalidate(vt.cache_key)
      vt.valid = False
    self._invalidator.force_invalidate(vts.cache_key)
    vts.valid = False

  def check(self,
            targets,
            partition_size_hint=None,
            target_colors=None,
            topological_order=False):
    """Checks whether each of the targets has changed and invalidates it if so.

    Returns a list of VersionedTargetSet objects (either valid or invalid). The returned sets
    'cover' the input targets, possibly partitioning them, with one caveat: if the FingerprintStrategy
    opted out of fingerprinting a target because it doesn't contribute to invalidation, then that
    target will be excluded from all_vts, invalid_vts, and the partitioned VTS.

    Callers can inspect these vts and rebuild the invalid ones, for example.

    If target_colors is specified, it must be a map from Target -> opaque 'color' values.
    Two Targets will be in the same partition only if they have the same color.
    """
    all_vts = self.wrap_targets(targets, topological_order=topological_order)
    invalid_vts = filter(lambda vt: not vt.valid, all_vts)
    return InvalidationCheck(all_vts, invalid_vts, partition_size_hint, target_colors)

  @property
  def task_name(self):
    return self._task_name

  def wrap_targets(self, targets, topological_order=False):
    """Wrap targets and their computed cache keys in VersionedTargets.

    If the FingerprintStrategy opted out of providing a fingerprint for a target, that target will not
    have an associated VersionedTarget returned.

    Returns a list of VersionedTargets, each representing one input target.
    """
    def vt_iter():
      if topological_order:
        sorted_targets = [t for t in reversed(sort_targets(targets)) if t in targets]
      else:
        sorted_targets = sorted(targets)
      for target in sorted_targets:
        target_key = self._key_for(target)
        if target_key is not None:
          yield VersionedTarget(self, target, target_key)
    return list(vt_iter())

  def previous_key(self, cache_key):
    return self._invalidator.previous_key(cache_key)

  def _key_for(self, target):
    try:
      return self._cache_key_generator.key_for_target(target,
                                                      transitive=self._invalidate_dependents,
                                                      fingerprint_strategy=self._fingerprint_strategy)
    except Exception as e:
      # This is a catch-all for problems we haven't caught up with and given a better diagnostic.
      # TODO(Eric Ayers): If you see this exception, add a fix to catch the problem earlier.
      exc_info = sys.exc_info()
      new_exception = self.CacheValidationError("Problem validating target {} in {}: {}"
                                                .format(target.id, target.address.spec_path, e))

      raise self.CacheValidationError, new_exception, exc_info[2]
Beispiel #15
0
 def invalidator(self):
   with temporary_dir() as root:
     yield BuildInvalidator(root)
Beispiel #16
0
class PythonChroot(object):
  _VALID_DEPENDENCIES = {
    Files: 'files',
    PrepCommand: 'prep',
    PythonLibrary: 'libraries',
    PythonRequirementLibrary: 'reqs',
    PythonBinary: 'binaries',
    PythonThriftLibrary: 'thrifts',
    PythonAntlrLibrary: 'antlrs',
    PythonTests: 'tests',
    Resources: 'resources'
  }

  class InvalidDependencyException(Exception):
    def __init__(self, target):
      super(PythonChroot.InvalidDependencyException, self).__init__(
        'Not a valid Python dependency! Found: {}'.format(target))

  @staticmethod
  def get_platforms(platform_list):
    return tuple({Platform.current() if p == 'current' else p for p in platform_list})

  def __init__(self,
               python_setup,
               python_repos,
               ivy_bootstrapper,
               thrift_binary_factory,
               interpreter,
               builder,
               targets,
               platforms,
               extra_requirements=None,
               log=None):
    self._python_setup = python_setup
    self._python_repos = python_repos
    self._ivy_bootstrapper = ivy_bootstrapper
    self._thrift_binary_factory = thrift_binary_factory

    self._interpreter = interpreter
    self._builder = builder
    self._targets = targets
    self._platforms = platforms
    self._extra_requirements = list(extra_requirements) if extra_requirements else []
    self._logger = log or logger

    # Note: unrelated to the general pants artifact cache.
    self._artifact_cache_root = os.path.join(
      self._python_setup.artifact_cache_dir, str(self._interpreter.identity))
    self._key_generator = CacheKeyGenerator()
    self._build_invalidator = BuildInvalidator(self._artifact_cache_root)

  def delete(self):
    """Deletes this chroot from disk if it has been dumped."""
    safe_rmtree(self.path())

  def debug(self, msg):
    self._logger.debug(msg)

  def path(self):
    return os.path.realpath(self._builder.path())

  def pex(self):
    return PEX(self.path(), interpreter=self._interpreter)

  def package_pex(self, filename):
    """Package into a PEX zipfile.

    :param filename: The filename where the PEX should be stored.
    """
    self._builder.build(filename)

  def _dump_library(self, library):
    def copy_to_chroot(base, path, add_function):
      src = os.path.join(get_buildroot(), base, path)
      add_function(src, path)

    self.debug('  Dumping library: {}'.format(library))
    for relpath in library.sources_relative_to_source_root():
      try:
        copy_to_chroot(library.target_base, relpath, self._builder.add_source)
      except OSError:
        logger.error("Failed to copy {path} for library {library}"
                     .format(path=os.path.join(library.target_base, relpath),
                             library=library))
        raise

    for resources_tgt in library.resources:
      for resource_file_from_source_root in resources_tgt.sources_relative_to_source_root():
        try:
          copy_to_chroot(resources_tgt.target_base, resource_file_from_source_root,
                         self._builder.add_resource)
        except OSError:
          logger.error("Failed to copy {path} for resource {resource}"
                       .format(path=os.path.join(resources_tgt.target_base,
                                                 resource_file_from_source_root),
                               resource=resources_tgt.address.spec))
          raise

  def _dump_requirement(self, req):
    self.debug('  Dumping requirement: {}'.format(req))
    self._builder.add_requirement(req)

  def _dump_distribution(self, dist):
    self.debug('  Dumping distribution: .../{}'.format(os.path.basename(dist.location)))
    self._builder.add_distribution(dist)

  def _generate_requirement(self, library, builder_cls):
    library_key = self._key_generator.key_for_target(library)
    builder = builder_cls(target=library,
                          root_dir=get_buildroot(),
                          target_suffix='-' + library_key.hash[:8])

    cache_dir = os.path.join(self._artifact_cache_root, library_key.id)
    if self._build_invalidator.needs_update(library_key):
      sdist = builder.build(interpreter=self._interpreter)
      safe_mkdir(cache_dir)
      shutil.copy(sdist, os.path.join(cache_dir, os.path.basename(sdist)))
      self._build_invalidator.update(library_key)

    return PythonRequirement(builder.requirement_string(), repository=cache_dir, use_2to3=True)

  def _generate_thrift_requirement(self, library):
    thrift_builder = functools.partial(PythonThriftBuilder,
                                       thrift_binary_factory=self._thrift_binary_factory,
                                       workdir=safe_mkdtemp(dir=self.path(), prefix='thrift.'))
    return self._generate_requirement(library, thrift_builder)

  def _generate_antlr_requirement(self, library):
    antlr_builder = functools.partial(PythonAntlrBuilder,
                                      ivy_bootstrapper=self._ivy_bootstrapper,
                                      workdir=safe_mkdtemp(dir=self.path(), prefix='antlr.'))
    return self._generate_requirement(library, antlr_builder)

  def resolve(self, targets):
    children = defaultdict(OrderedSet)

    def add_dep(trg):
      for target_type, target_key in self._VALID_DEPENDENCIES.items():
        if isinstance(trg, target_type):
          children[target_key].add(trg)
          return
        elif type(trg) == Target:
          return
      raise self.InvalidDependencyException(trg)
    for target in targets:
      target.walk(add_dep)
    return children

  def dump(self):
    self.debug('Building chroot for {}:'.format(self._targets))
    targets = self.resolve(self._targets)

    for lib in targets['libraries'] | targets['binaries']:
      self._dump_library(lib)

    generated_reqs = OrderedSet()
    if targets['thrifts']:
      for thr in targets['thrifts']:
        generated_reqs.add(self._generate_thrift_requirement(thr))
      generated_reqs.add(PythonRequirement('thrift', use_2to3=True))

    for antlr in targets['antlrs']:
      generated_reqs.add(self._generate_antlr_requirement(antlr))

    reqs_from_libraries = OrderedSet()
    for req_lib in targets['reqs']:
      for req in req_lib.payload.requirements:
        reqs_from_libraries.add(req)

    reqs_to_build = OrderedSet()
    find_links = OrderedSet()

    for req in reqs_from_libraries | generated_reqs | self._extra_requirements:
      if not req.should_build(self._interpreter.python, Platform.current()):
        self.debug('Skipping {} based upon version filter'.format(req))
        continue
      reqs_to_build.add(req)
      self._dump_requirement(req.requirement)
      if req.repository:
        find_links.add(req.repository)

    distributions = self._resolve_multi(reqs_to_build, find_links)

    locations = set()
    for platform, dist_set in distributions.items():
      for dist in dist_set:
        if dist.location not in locations:
          self._dump_distribution(dist)
        locations.add(dist.location)

    if len(targets['binaries']) > 1:
      print('WARNING: Target has multiple python_binary targets!', file=sys.stderr)

    return self._builder

  def _resolve_multi(self, requirements, find_links):
    """Multi-platform dependency resolution for PEX files.

       Given a pants configuration and a set of requirements, return a map of platform name -> list
       of :class:`pkg_resources.Distribution` instances needed to satisfy them on that platform.
       That may involve distributions for multiple platforms.

       :param requirements: A list of :class:`PythonRequirement` objects to resolve.
       :param find_links: Additional paths to search for source packages during resolution.
    """
    distributions = dict()
    platforms = self.get_platforms(self._platforms or self._python_setup.platforms)
    fetchers = self._python_repos.get_fetchers()
    fetchers.extend(Fetcher([path]) for path in find_links)
    context = self._python_repos.get_network_context()

    for platform in platforms:
      requirements_cache_dir = os.path.join(self._python_setup.resolver_cache_dir,
                                            str(self._interpreter.identity))
      distributions[platform] = resolve(
        requirements=[req.requirement for req in requirements],
        interpreter=self._interpreter,
        fetchers=fetchers,
        platform=platform,
        context=context,
        cache=requirements_cache_dir,
        cache_ttl=self._python_setup.resolver_cache_ttl,
        allow_prereleases=self._python_setup.resolver_allow_prereleases)

    return distributions
Beispiel #17
0
class PythonChroot(object):
    _VALID_DEPENDENCIES = {
        PrepCommand: 'prep',
        PythonLibrary: 'libraries',
        PythonRequirementLibrary: 'reqs',
        PythonBinary: 'binaries',
        PythonThriftLibrary: 'thrifts',
        PythonAntlrLibrary: 'antlrs',
        PythonTests: 'tests',
        Resources: 'resources'
    }

    class InvalidDependencyException(Exception):
        def __init__(self, target):
            super(PythonChroot.InvalidDependencyException, self).__init__(
                'Not a valid Python dependency! Found: {}'.format(target))

    @staticmethod
    def get_platforms(platform_list):
        return tuple({
            Platform.current() if p == 'current' else p
            for p in platform_list
        })

    def __init__(self,
                 python_setup,
                 python_repos,
                 ivy_bootstrapper,
                 thrift_binary_factory,
                 interpreter,
                 builder,
                 targets,
                 platforms,
                 extra_requirements=None,
                 log=None):
        self._python_setup = python_setup
        self._python_repos = python_repos
        self._ivy_bootstrapper = ivy_bootstrapper
        self._thrift_binary_factory = thrift_binary_factory

        self._interpreter = interpreter
        self._builder = builder
        self._targets = targets
        self._platforms = platforms
        self._extra_requirements = list(
            extra_requirements) if extra_requirements else []
        self._logger = log or logger

        # Note: unrelated to the general pants artifact cache.
        self._artifact_cache_root = os.path.join(
            self._python_setup.artifact_cache_dir,
            str(self._interpreter.identity))
        self._key_generator = CacheKeyGenerator()
        self._build_invalidator = BuildInvalidator(self._artifact_cache_root)

    def delete(self):
        """Deletes this chroot from disk if it has been dumped."""
        safe_rmtree(self.path())

    def debug(self, msg):
        self._logger.debug(msg)

    def path(self):
        return os.path.realpath(self._builder.path())

    def pex(self):
        return PEX(self.path(), interpreter=self._interpreter)

    def package_pex(self, filename):
        """Package into a PEX zipfile.

    :param filename: The filename where the PEX should be stored.
    """
        self._builder.build(filename)

    def _dump_library(self, library):
        def copy_to_chroot(base, path, add_function):
            src = os.path.join(get_buildroot(), base, path)
            add_function(src, path)

        self.debug('  Dumping library: {}'.format(library))
        for relpath in library.sources_relative_to_source_root():
            try:
                copy_to_chroot(library.target_base, relpath,
                               self._builder.add_source)
            except OSError:
                logger.error(
                    "Failed to copy {path} for library {library}".format(
                        path=os.path.join(library.target_base, relpath),
                        library=library))
                raise

        for resources_tgt in library.resources:
            for resource_file_from_source_root in resources_tgt.sources_relative_to_source_root(
            ):
                try:
                    copy_to_chroot(resources_tgt.target_base,
                                   resource_file_from_source_root,
                                   self._builder.add_resource)
                except OSError:
                    logger.error(
                        "Failed to copy {path} for resource {resource}".format(
                            path=os.path.join(resources_tgt.target_base,
                                              resource_file_from_source_root),
                            resource=resources_tgt.address.spec))
                    raise

    def _dump_requirement(self, req):
        self.debug('  Dumping requirement: {}'.format(req))
        self._builder.add_requirement(req)

    def _dump_distribution(self, dist):
        self.debug('  Dumping distribution: .../{}'.format(
            os.path.basename(dist.location)))
        self._builder.add_distribution(dist)

    def _generate_requirement(self, library, builder_cls):
        library_key = self._key_generator.key_for_target(library)
        builder = builder_cls(target=library,
                              root_dir=get_buildroot(),
                              target_suffix='-' + library_key.hash[:8])

        cache_dir = os.path.join(self._artifact_cache_root, library_key.id)
        if self._build_invalidator.needs_update(library_key):
            sdist = builder.build(interpreter=self._interpreter)
            safe_mkdir(cache_dir)
            shutil.copy(sdist, os.path.join(cache_dir,
                                            os.path.basename(sdist)))
            self._build_invalidator.update(library_key)

        return PythonRequirement(builder.requirement_string(),
                                 repository=cache_dir,
                                 use_2to3=True)

    def _generate_thrift_requirement(self, library):
        thrift_builder = functools.partial(
            PythonThriftBuilder,
            thrift_binary_factory=self._thrift_binary_factory,
            workdir=safe_mkdtemp(dir=self.path(), prefix='thrift.'))
        return self._generate_requirement(library, thrift_builder)

    def _generate_antlr_requirement(self, library):
        antlr_builder = functools.partial(
            PythonAntlrBuilder,
            ivy_bootstrapper=self._ivy_bootstrapper,
            workdir=safe_mkdtemp(dir=self.path(), prefix='antlr.'))
        return self._generate_requirement(library, antlr_builder)

    def resolve(self, targets):
        children = defaultdict(OrderedSet)

        def add_dep(trg):
            # Currently we handle all of our code generation, so we don't want to operate over any
            # synthetic targets injected upstream.
            # TODO(John Sirois): Revisit this when building a proper python product pipeline.
            if trg.is_synthetic:
                return

            for target_type, target_key in self._VALID_DEPENDENCIES.items():
                if isinstance(trg, target_type):
                    children[target_key].add(trg)
                    return
                elif type(trg) == Target:
                    return
            raise self.InvalidDependencyException(trg)

        for target in targets:
            target.walk(add_dep)
        return children

    def dump(self):
        self.debug('Building chroot for {}:'.format(self._targets))
        targets = self.resolve(self._targets)

        for lib in targets['libraries'] | targets['binaries']:
            self._dump_library(lib)

        generated_reqs = OrderedSet()
        if targets['thrifts']:
            for thr in targets['thrifts']:
                generated_reqs.add(self._generate_thrift_requirement(thr))
            generated_reqs.add(PythonRequirement('thrift', use_2to3=True))

        for antlr in targets['antlrs']:
            generated_reqs.add(self._generate_antlr_requirement(antlr))

        reqs_from_libraries = OrderedSet()
        for req_lib in targets['reqs']:
            for req in req_lib.payload.requirements:
                reqs_from_libraries.add(req)

        reqs_to_build = OrderedSet()
        find_links = OrderedSet()

        for req in reqs_from_libraries | generated_reqs | self._extra_requirements:
            if not req.should_build(self._interpreter.python,
                                    Platform.current()):
                self.debug('Skipping {} based upon version filter'.format(req))
                continue
            reqs_to_build.add(req)
            self._dump_requirement(req.requirement)
            if req.repository:
                find_links.add(req.repository)

        distributions = self._resolve_multi(reqs_to_build, find_links)

        locations = set()
        for platform, dist_set in distributions.items():
            for dist in dist_set:
                if dist.location not in locations:
                    self._dump_distribution(dist)
                locations.add(dist.location)

        if len(targets['binaries']) > 1:
            print('WARNING: Target has multiple python_binary targets!',
                  file=sys.stderr)

        return self._builder

    def _resolve_multi(self, requirements, find_links):
        """Multi-platform dependency resolution for PEX files.

       Given a pants configuration and a set of requirements, return a list of distributions
       that must be included in order to satisfy them.  That may involve distributions for
       multiple platforms.

       :param requirements: A list of :class:`PythonRequirement` objects to resolve.
       :param find_links: Additional paths to search for source packages during resolution.
    """
        distributions = dict()
        platforms = self.get_platforms(self._platforms
                                       or self._python_setup.platforms)
        fetchers = self._python_repos.get_fetchers()
        fetchers.extend(Fetcher([path]) for path in find_links)
        context = self._python_repos.get_network_context()

        for platform in platforms:
            requirements_cache_dir = os.path.join(
                self._python_setup.resolver_cache_dir,
                str(self._interpreter.identity))
            distributions[platform] = resolve(
                requirements=[req.requirement for req in requirements],
                interpreter=self._interpreter,
                fetchers=fetchers,
                platform=platform,
                context=context,
                cache=requirements_cache_dir,
                cache_ttl=self._python_setup.resolver_cache_ttl)

        return distributions