Example #1
0
 def combine_versioned_targets(vts):
   targets = []
   for vt in vts:
     targets.extend(vt.targets)
   cache_key = CacheKeyGenerator.combine_cache_keys([vt.cache_key for vt in vts])
   valid = all([vt.valid for vt in vts])
   return VersionedTargetSet(targets, cache_key, valid)
Example #2
0
  def ivy_resolve(self, targets, java_runner=None, symlink_ivyxml=False, silent=False,
                  workunit_name=None, workunit_labels=None):
    java_runner = java_runner or runjava_indivisible

    targets = set(targets)

    if not targets:
      return []
    
    work_dir = self.context.config.get('ivy-resolve', 'workdir')
    confs = self.context.config.getlist('ivy-resolve', 'confs')

    with self.invalidated(targets,
                          only_buildfiles=True,
                          invalidate_dependents=True,
                          silent=silent) as invalidation_check:
      global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
      target_workdir = os.path.join(work_dir, global_vts.cache_key.hash)
      target_classpath_file = os.path.join(target_workdir, 'classpath')
      raw_target_classpath_file = target_classpath_file + '.raw'
      raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp'
      symlink_dir = os.path.join(target_workdir, 'jars')

      # Note that it's possible for all targets to be valid but for no classpath file to exist at
      # target_classpath_file, e.g., if we previously built a superset of targets.
      if invalidation_check.invalid_vts or not os.path.exists(raw_target_classpath_file):
        ivy_utils = IvyUtils(config=self.context.config,
                             options=self.context.options,
                             log=self.context.log)
        args = (['-cachepath', raw_target_classpath_file_tmp] +
                ['-confs'] + confs)

        def exec_ivy():
          ivy_utils.exec_ivy(
            target_workdir=target_workdir,
            targets=targets,
            args=args,
            runjava=java_runner,
            workunit_name='ivy',
            workunit_factory=self.context.new_workunit,
            symlink_ivyxml=symlink_ivyxml,
          )

        if workunit_name:
          with self.context.new_workunit(name=workunit_name, labels=workunit_labels or []):
            exec_ivy()
        else:
          exec_ivy()

        if not os.path.exists(raw_target_classpath_file_tmp):
          raise TaskError('Ivy failed to create classpath file at %s' % raw_target_classpath_file_tmp)
        shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file)

        if self.artifact_cache_writes_enabled():
          self.update_artifact_cache([(global_vts, [raw_target_classpath_file])])

    # Make our actual classpath be symlinks, so that the paths are uniform across systems.
    # Note that we must do this even if we read the raw_target_classpath_file from the artifact
    # cache. If we cache the target_classpath_file we won't know how to create the symlinks.
    symlink_map = IvyUtils.symlink_cachepath(raw_target_classpath_file, symlink_dir, target_classpath_file)
    with Task.symlink_map_lock:
      all_symlinks_map = self.context.products.get_data('symlink_map') or defaultdict(list)
      for path, symlink in symlink_map.items():
        all_symlinks_map[os.path.realpath(path)].append(symlink)
      self.context.products.set_data('symlink_map', all_symlinks_map)

    with IvyUtils.cachepath(target_classpath_file) as classpath:
      stripped_classpath = [path.strip() for path in classpath]
      return [path for path in stripped_classpath if IvyUtils.is_mappable_artifact(path)]
Example #3
0
 def check_artifact_cache_for(self, invalidation_check):
   # Ivy resolution is an output dependent on the entire target set, and is not divisible
   # by target. So we can only cache it keyed by the entire target set.
   global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
   return [global_vts]
Example #4
0
 def check_artifact_cache_for(self, invalidation_check):
     # Ivy resolution is an output dependent on the entire target set, and is not divisible
     # by target. So we can only cache it keyed by the entire target set.
     global_vts = VersionedTargetSet.from_versioned_targets(
         invalidation_check.all_vts)
     return [global_vts]
Example #5
0
    def execute(self, targets):
        """Resolves the specified confs for the configured targets and returns an iterator over
    tuples of (conf, jar path).
    """
        def dirname_for_requested_targets(targets):
            """Where we put the classpath file for this set of targets."""
            sha = hashlib.sha1()
            for t in targets:
                sha.update(t.id)
            return sha.hexdigest()

        def is_classpath(target):
            return is_jar(target) or (is_internal(target) and any(
                jar for jar in target.jar_dependencies if jar.rev))

        groups = self.context.products.get_data('exclusives_groups')

        # Below, need to take the code that actually execs ivy, and invoke it once for each
        # group. Then after running ivy, we need to take the resulting classpath, and load it into
        # the build products.

        # The set of groups we need to consider is complicated:
        # - If there are no conflicting exclusives (ie, there's only one entry in the map),
        #   then we just do the one.
        # - If there are conflicts, then there will be at least three entries in the groups map:
        #   - the group with no exclusives (X)
        #   - the two groups that are in conflict (A and B).
        # In the latter case, we need to do the resolve twice: Once for A+X, and once for B+X,
        # because things in A and B can depend on things in X; and so they can indirectly depend
        # on the dependencies of X. (I think this well be covered by the computed transitive dependencies of
        # A and B. But before pushing this change, review this comment, and make sure that this is
        # working correctly.
        for group_key in groups.get_group_keys():
            # Narrow the groups target set to just the set of targets that we're supposed to build.
            # Normally, this shouldn't be different from the contents of the group.
            group_targets = groups.get_targets_for_group_key(group_key) & set(
                targets)

            classpath_targets = OrderedSet()
            for target in group_targets:
                classpath_targets.update(
                    filter(is_classpath, filter(is_concrete,
                                                target.resolve())))

            if len(classpath_targets) == 0:
                continue  # Nothing to do.

            target_workdir = os.path.join(
                self._work_dir, dirname_for_requested_targets(group_targets))
            target_classpath_file = os.path.join(target_workdir, 'classpath')
            with self.invalidated(
                    classpath_targets,
                    only_buildfiles=True,
                    invalidate_dependents=True) as invalidation_check:
                # Note that it's possible for all targets to be valid but for no classpath file to exist at
                # target_classpath_file, e.g., if we previously built a superset of targets.
                if invalidation_check.invalid_vts or not os.path.exists(
                        target_classpath_file):
                    # TODO(benjy): s/targets/classpath_targets/ ??
                    self._exec_ivy(
                        target_workdir, targets,
                        ['-cachepath', target_classpath_file, '-confs'] +
                        self._confs)

                    if not os.path.exists(target_classpath_file):
                        raise TaskError(
                            'Ivy failed to create classpath file at %s %s' %
                            target_classpath_file)
                    if self.get_artifact_cache(
                    ) and self.context.options.write_to_artifact_cache:
                        global_vts = VersionedTargetSet.from_versioned_targets(
                            invalidation_check.all_vts)
                        self.update_artifact_cache([(global_vts,
                                                     [target_classpath_file])])

            with self._cachepath(target_classpath_file) as classpath:
                for path in classpath:
                    if self._map_jar(path):
                        for conf in self._confs:
                            groups.update_compatible_classpaths(
                                group_key, [(conf, path.strip())])

        if self._report:
            self._generate_ivy_report()

        if self.context.products.isrequired("ivy_jar_products"):
            self._populate_ivy_jar_products()

        create_jardeps_for = self.context.products.isrequired(
            self._mapfor_typename())
        if create_jardeps_for:
            genmap = self.context.products.get(self._mapfor_typename())
            for target in filter(create_jardeps_for, targets):
                self._mapjars(genmap, target)
Example #6
0
    def ivy_resolve(self,
                    targets,
                    java_runner=None,
                    ivy_args=None,
                    symlink_ivyxml=False,
                    silent=False,
                    workunit_name=None,
                    workunit_labels=None):
        java_runner = java_runner or runjava_indivisible

        ivy_args = ivy_args or []

        targets = set(targets)

        if not targets:
            return []

        work_dir = self.context.config.get('ivy-resolve', 'workdir')
        confs = self.context.config.getlist('ivy-resolve', 'confs')

        with self.invalidated(targets,
                              only_buildfiles=True,
                              invalidate_dependents=True,
                              silent=silent) as invalidation_check:
            global_vts = VersionedTargetSet.from_versioned_targets(
                invalidation_check.all_vts)
            target_workdir = os.path.join(work_dir, global_vts.cache_key.hash)
            target_classpath_file = os.path.join(target_workdir, 'classpath')
            target_classpath_file_tmp = target_classpath_file + '.tmp'
            # Note that it's possible for all targets to be valid but for no classpath file to exist at
            # target_classpath_file, e.g., if we previously built a superset of targets.
            if invalidation_check.invalid_vts or not os.path.exists(
                    target_classpath_file):
                ivy_utils = IvyUtils(config=self.context.config,
                                     options=self.context.options,
                                     log=self.context.log)
                args = (['-cachepath', target_classpath_file_tmp] +
                        ['-confs'] + confs + ivy_args)

                def exec_ivy():
                    ivy_utils.exec_ivy(
                        target_workdir=target_workdir,
                        targets=targets,
                        args=args,
                        runjava=java_runner,
                        workunit_name='ivy',
                        workunit_factory=self.context.new_workunit,
                        symlink_ivyxml=symlink_ivyxml,
                    )

                if workunit_name:
                    with self.context.new_workunit(name=workunit_name,
                                                   labels=workunit_labels
                                                   or []):
                        exec_ivy()
                else:
                    exec_ivy()

                if not os.path.exists(target_classpath_file_tmp):
                    raise TaskError(
                        'Ivy failed to create classpath file at %s' %
                        target_classpath_file_tmp)
                shutil.move(target_classpath_file_tmp, target_classpath_file)
                if self.get_artifact_cache(
                ) and self.context.options.write_to_artifact_cache:
                    self.update_artifact_cache([(global_vts,
                                                 [target_classpath_file])])

        with IvyUtils.cachepath(target_classpath_file) as classpath:
            stripped_classpath = [path.strip() for path in classpath]
            return [
                path for path in stripped_classpath
                if IvyUtils.is_mappable_artifact(path)
            ]
Example #7
0
  def execute(self, targets):
    """Resolves the specified confs for the configured targets and returns an iterator over
    tuples of (conf, jar path).
    """
    def dirname_for_requested_targets(targets):
      """Where we put the classpath file for this set of targets."""
      sha = hashlib.sha1()
      for t in targets:
        sha.update(t.id)
      return sha.hexdigest()

    def is_classpath(target):
      return is_jar(target) or (
        is_internal(target) and any(jar for jar in target.jar_dependencies if jar.rev)
      )

    groups = self.context.products.get_data('exclusives_groups')

    # Below, need to take the code that actually execs ivy, and invoke it once for each
    # group. Then after running ivy, we need to take the resulting classpath, and load it into
    # the build products.

    # The set of groups we need to consider is complicated:
    # - If there are no conflicting exclusives (ie, there's only one entry in the map),
    #   then we just do the one.
    # - If there are conflicts, then there will be at least three entries in the groups map:
    #   - the group with no exclusives (X)
    #   - the two groups that are in conflict (A and B).
    # In the latter case, we need to do the resolve twice: Once for A+X, and once for B+X,
    # because things in A and B can depend on things in X; and so they can indirectly depend
    # on the dependencies of X. (I think this well be covered by the computed transitive dependencies of
    # A and B. But before pushing this change, review this comment, and make sure that this is
    # working correctly.
    for group_key in groups.get_group_keys():
      # Narrow the groups target set to just the set of targets that we're supposed to build.
      # Normally, this shouldn't be different from the contents of the group.
      group_targets = groups.get_targets_for_group_key(group_key) & set(targets)

      classpath_targets = OrderedSet()
      for target in group_targets:
        classpath_targets.update(filter(is_classpath, filter(is_concrete, target.resolve())))

      if len(classpath_targets) == 0:
        continue  # Nothing to do.

      target_workdir = os.path.join(self._work_dir, dirname_for_requested_targets(group_targets))
      target_classpath_file = os.path.join(target_workdir, 'classpath')
      with self.invalidated(classpath_targets, only_buildfiles=True,
                            invalidate_dependents=True) as invalidation_check:
        # Note that it's possible for all targets to be valid but for no classpath file to exist at
        # target_classpath_file, e.g., if we previously built a superset of targets.
        if invalidation_check.invalid_vts or not os.path.exists(target_classpath_file):
          # TODO(benjy): s/targets/classpath_targets/ ??
          self._exec_ivy(target_workdir, targets, [
            '-cachepath', target_classpath_file,
            '-confs'
          ] + self._confs)

          if not os.path.exists(target_classpath_file):
            raise TaskError('Ivy failed to create classpath file at %s %s' % target_classpath_file)
          if self.get_artifact_cache() and self.context.options.write_to_artifact_cache:
            global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)
            self.update_artifact_cache([(global_vts, [target_classpath_file])])

      with self._cachepath(target_classpath_file) as classpath:
        for path in classpath:
          if self._map_jar(path):
            for conf in self._confs:
              groups.update_compatible_classpaths(group_key, [(conf, path.strip())])

    if self._report:
      self._generate_ivy_report()

    if self.context.products.isrequired("ivy_jar_products"):
      self._populate_ivy_jar_products()

    create_jardeps_for = self.context.products.isrequired(self._mapfor_typename())
    if create_jardeps_for:
      genmap = self.context.products.get(self._mapfor_typename())
      for target in filter(create_jardeps_for, targets):
        self._mapjars(genmap, target)