Пример #1
0
  def run(self, lock):
    with self.check_errors("Target contains a dependency cycle") as error:
      for target in self.targets:
        try:
          InternalTarget.check_cycles(target)
        except InternalTarget.CycleException as e:
          error(target.id)

    timer = None
    if self.options.time:
      class Timer(object):
        def now(self):
          return time.time()
        def log(self, message):
          print(message)
      timer = Timer()

    logger = None
    if self.options.log or self.options.log_level:
      from twitter.common.log import init
      from twitter.common.log.options import LogOptions
      LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper())
      logdir = self.options.logdir or self.config.get('goals', 'logdir', default=None)
      if logdir:
        safe_mkdir(logdir)
        LogOptions.set_log_dir(logdir)
        init('goals')
      else:
        init()
      logger = log

    if self.options.recursive_directory:
      log.warn('--all-recursive is deprecated, use a target spec with the form [dir]:: instead')
      for dir in self.options.recursive_directory:
        self.add_target_recursive(dir)

    if self.options.target_directory:
      log.warn('--all is deprecated, use a target spec with the form [dir]: instead')
      for dir in self.options.target_directory:
        self.add_target_directory(dir)

    context = Context(self.config, self.options, self.targets, lock=lock, log=logger)

    unknown = []
    for phase in self.phases:
      if not phase.goals():
        unknown.append(phase)

    if unknown:
        print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown))
        print('')
        return Phase.execute(context, 'goals')

    if logger:
      logger.debug('Operating on targets: %s', self.targets)

    return Phase.attempt(context, self.phases, timer=timer)
Пример #2
0
  def testSort(self):
    a = MockTarget('a', [])
    b = MockTarget('b', [a])
    c = MockTarget('c', [b])
    d = MockTarget('d', [c, a])
    e = MockTarget('e', [d])

    self.assertEquals(InternalTarget.sort_targets([a,b,c,d,e]), [e,d,c,b,a])
    self.assertEquals(InternalTarget.sort_targets([b,d,a,e,c]), [e,d,c,b,a])
    self.assertEquals(InternalTarget.sort_targets([e,d,c,b,a]), [e,d,c,b,a])
Пример #3
0
 def __init__(self, name, dependencies=None, num_sources=0, exclusives=None):
   with ParseContext.temp():
     InternalTarget.__init__(self, name, dependencies, exclusives=exclusives)
     TargetWithSources.__init__(self, name, exclusives=exclusives)
   self.num_sources = num_sources
   self.declared_exclusives = defaultdict(set)
   if exclusives is not None:
     for k in exclusives:
       self.declared_exclusives[k] = set([exclusives[k]])
   self.exclusives = None
Пример #4
0
    def run(self, lock):
        with self.check_errors("Target contains a dependency cycle") as error:
            with self.timer.timing("parse:check_cycles"):
                for target in self.targets:
                    try:
                        InternalTarget.check_cycles(target)
                    except InternalTarget.CycleException as e:
                        error(target.id)

        logger = None
        if self.options.log or self.options.log_level:
            from twitter.common.log import init
            from twitter.common.log.options import LogOptions

            LogOptions.set_stderr_log_level((self.options.log_level or "info").upper())
            logdir = self.options.logdir or self.config.get("goals", "logdir", default=None)
            if logdir:
                safe_mkdir(logdir)
                LogOptions.set_log_dir(logdir)
                init("goals")
            else:
                init()
            logger = log

        if self.options.recursive_directory:
            log.warn("--all-recursive is deprecated, use a target spec with the form [dir]:: instead")
            for dir in self.options.recursive_directory:
                self.add_target_recursive(dir)

        if self.options.target_directory:
            log.warn("--all is deprecated, use a target spec with the form [dir]: instead")
            for dir in self.options.target_directory:
                self.add_target_directory(dir)

        context = Context(self.config, self.options, self.targets, lock=lock, log=logger)

        unknown = []
        for phase in self.phases:
            if not phase.goals():
                unknown.append(phase)

        if unknown:
            print("Unknown goal(s): %s" % " ".join(phase.name for phase in unknown))
            print("")
            return Phase.execute(context, "goals")

        if logger:
            logger.debug("Operating on targets: %s", self.targets)

        ret = Phase.attempt(context, self.phases, timer=self.timer if self.options.time else None)
        if self.options.time:
            print("Timing report")
            print("=============")
            self.timer.print_timings()
        return ret
Пример #5
0
  def run(self, lock):
    with self.check_errors("Target contains a dependency cycle") as error:
      with self.timer.timing('parse:check_cycles'):
        for target in self.targets:
          try:
            InternalTarget.check_cycles(target)
          except InternalTarget.CycleException as e:
            error(target.id)

    logger = None
    if self.options.log or self.options.log_level:
      from twitter.common.log import init
      from twitter.common.log.options import LogOptions
      LogOptions.set_stderr_log_level((self.options.log_level or 'info').upper())
      logdir = self.options.logdir or self.config.get('goals', 'logdir', default=None)
      if logdir:
        safe_mkdir(logdir)
        LogOptions.set_log_dir(logdir)
        init('goals')
      else:
        init()
      logger = log

    if self.options.recursive_directory:
      log.warn('--all-recursive is deprecated, use a target spec with the form [dir]:: instead')
      for dir in self.options.recursive_directory:
        self.add_target_recursive(dir)

    if self.options.target_directory:
      log.warn('--all is deprecated, use a target spec with the form [dir]: instead')
      for dir in self.options.target_directory:
        self.add_target_directory(dir)

    context = Context(self.config, self.options, self.targets, lock=lock, log=logger)

    unknown = []
    for phase in self.phases:
      if not phase.goals():
        unknown.append(phase)

    if unknown:
        print('Unknown goal(s): %s' % ' '.join(phase.name for phase in unknown))
        print('')
        return Phase.execute(context, 'goals')

    if logger:
      logger.debug('Operating on targets: %s', self.targets)

    ret = Phase.attempt(context, self.phases, timer=self.timer if self.options.time else None)
    if self.options.time:
      print('Timing report')
      print('=============')
      self.timer.print_timings()
    return ret
Пример #6
0
  def test_detect_cycle_direct(self):
    a = MockTarget('a')

    # no cycles yet
    InternalTarget.sort_targets([a])
    a.update_dependencies([a])
    try:
      InternalTarget.sort_targets([a])
      self.fail("Expected a cycle to be detected")
    except InternalTarget.CycleException:
      # expected
      pass
Пример #7
0
  def testDetectCycleDirect(self):
    a = MockTarget('a')

    # no cycles yet
    InternalTarget.check_cycles(a)
    a.internal_dependencies = [ a ]
    try:
      InternalTarget.check_cycles(a)
      self.fail("Expected a cycle to be detected")
    except InternalTarget.CycleException:
      # expected
      pass
Пример #8
0
    def test_detect_cycle_direct(self):
        a = MockTarget('a')

        # no cycles yet
        InternalTarget.sort_targets([a])
        a.update_dependencies([a])
        try:
            InternalTarget.sort_targets([a])
            self.fail("Expected a cycle to be detected")
        except InternalTarget.CycleException:
            # expected
            pass
Пример #9
0
  def testDetectCycleDirect(self):
    a = MockTarget('a')

    # no cycles yet
    InternalTarget.sort_targets([a])
    a.internal_dependencies = [a]
    try:
      InternalTarget.sort_targets([a])
      self.fail("Expected a cycle to be detected")
    except InternalTarget.CycleException:
      # expected
      pass
Пример #10
0
    def testDetectCycleDirect(self):
        a = MockTarget("a")

        # no cycles yet
        InternalTarget.check_cycles(a)
        a.internal_dependencies = [a]
        try:
            InternalTarget.check_cycles(a)
            self.fail("Expected a cycle to be detected")
        except InternalTarget.CycleException:
            # expected
            pass
Пример #11
0
    def test_validation(self):
        with ParseContext.temp('InternalTargetTest/test_validation'):
            InternalTarget(name="valid", dependencies=None)
            self.assertRaises(TargetDefinitionException,
                              InternalTarget,
                              name=1,
                              dependencies=None)

            InternalTarget(name="valid2", dependencies=Target(name='mybird'))
            self.assertRaises(TargetDefinitionException,
                              InternalTarget,
                              name='valid3',
                              dependencies=1)
Пример #12
0
  def test_detect_cycle_indirect(self):
    c = MockTarget('c')
    b = MockTarget('b', [c])
    a = MockTarget('a', [c, b])

    # no cycles yet
    InternalTarget.sort_targets([a])

    c.internal_dependencies = [a]
    try:
      InternalTarget.sort_targets([a])
      self.fail("Expected a cycle to be detected")
    except InternalTarget.CycleException:
      # expected
      pass
Пример #13
0
  def testDetectIndirect(self):
    c = MockTarget('c')
    b = MockTarget('b', c)
    a = MockTarget('a', c, b)

    # no cycles yet
    InternalTarget.sort_targets([a])

    c.internal_dependencies = [a]
    try:
      InternalTarget.sort_targets([a])
      self.fail("Expected a cycle to be detected")
    except InternalTarget.CycleException:
      # expected
      pass
Пример #14
0
    def testDetectIndirect(self):
        c = MockTarget("c")
        b = MockTarget("b", c)
        a = MockTarget("a", c, b)

        # no cycles yet
        InternalTarget.check_cycles(a)

        c.internal_dependencies = [a]
        try:
            InternalTarget.check_cycles(a)
            self.fail("Expected a cycle to be detected")
        except InternalTarget.CycleException:
            # expected
            pass
Пример #15
0
  def testDetectIndirect(self):
    c = MockTarget('c')
    b = MockTarget('b', c)
    a = MockTarget('a', c, b)

    # no cycles yet
    InternalTarget.check_cycles(a)

    c.internal_dependencies = [ a ]
    try:
      InternalTarget.check_cycles(a)
      self.fail("Expected a cycle to be detected")
    except InternalTarget.CycleException:
      # expected
      pass
Пример #16
0
  def create_chunks(context, goals):
    def discriminator(target):
      for i, goal in enumerate(goals):
        if goal.group.predicate(target):
          return i
      return 'other'

    # TODO(John Sirois): coalescing should be made available in another spot, InternalTarget is jvm
    # specific, and all we care is that the Targets have dependencies defined
    coalesced = InternalTarget.coalesce_targets(context.targets(is_internal), discriminator)
    coalesced = list(reversed(coalesced))

    def not_internal(target):
      return not is_internal(target)
    rest = OrderedSet(context.targets(not_internal))

    chunks = [rest] if rest else []
    flavor = None
    chunk_start = 0
    for i, target in enumerate(coalesced):
      target_flavor = discriminator(target)
      if target_flavor != flavor and i > chunk_start:
        chunks.append(OrderedSet(coalesced[chunk_start:i]))
        chunk_start = i
      flavor = target_flavor
    if chunk_start < len(coalesced):
      chunks.append(OrderedSet(coalesced[chunk_start:]))

    context.log.debug('::: created chunks(%d)' % len(chunks))
    for i, chunk in enumerate(chunks):
      context.log.debug('  chunk(%d):\n\t%s' % (i, '\n\t'.join(sorted(map(str, chunk)))))

    return chunks
Пример #17
0
    def exported_targets(self):
        candidates = set()
        if self.transitive:
            candidates.update(self.context.targets())
        else:
            candidates.update(self.context.target_roots)

            def get_synthetic(lang, target):
                mappings = self.context.products.get(lang).get(target)
                if mappings:
                    for generated in mappings.itervalues():
                        for synthetic in generated:
                            yield synthetic

            # Handle the case where a code gen target is in the listed roots and the thus the publishable
            # target is a synthetic twin generated by a code gen task upstream.
            for candidate in self.context.target_roots:
                candidates.update(get_synthetic('java', candidate))
                candidates.update(get_synthetic('scala', candidate))

        def exportable(tgt):
            return tgt in candidates and tgt.is_exported

        return OrderedSet(
            filter(
                exportable,
                reversed(
                    InternalTarget.sort_targets(filter(exportable,
                                                       candidates)))))
Пример #18
0
  def create_chunks(context, goals):
    def discriminator(target):
      for i, goal in enumerate(goals):
        if goal.group.predicate(target):
          return i
      return 'other'

    # TODO(John Sirois): coalescing should be made available in another spot, InternalTarget is jvm
    # specific, and all we care is that the Targets have dependencies defined
    coalesced = InternalTarget.coalesce_targets(context.targets(is_internal), discriminator)
    coalesced = list(reversed(coalesced))

    def not_internal(target):
      return not is_internal(target)
    rest = OrderedSet(context.targets(not_internal))

    chunks = [rest] if rest else []
    flavor = None
    chunk_start = 0
    for i, target in enumerate(coalesced):
      target_flavor = discriminator(target)
      if target_flavor != flavor and i > chunk_start:
        chunks.append(OrderedSet(coalesced[chunk_start:i]))
        chunk_start = i
      flavor = target_flavor
    if chunk_start < len(coalesced):
      chunks.append(OrderedSet(coalesced[chunk_start:]))

    context.log.debug('::: created chunks(%d)' % len(chunks))
    for i, chunk in enumerate(chunks):
      context.log.debug('  chunk(%d):\n\t%s' % (i, '\n\t'.join(sorted(map(str, chunk)))))

    return chunks
Пример #19
0
  def _create_chunks(context, goals):

    def discriminator(target):
      for i, goal in enumerate(goals):
        if goal.group.predicate(target):
          return i
      return 'other'

    # First, divide the set of all targets to be built into compatible chunks, based
    # on their declared exclusives. Then, for each chunk of compatible exclusives, do
    # further subchunking. At the end, we'll have a list of chunks to be built,
    # which will go through the chunks of each exclusives-compatible group separately.

    # TODO(markcc); chunks with incompatible exclusives require separate ivy resolves.
    # Either interleave the ivy task in this group so that it runs once for each batch of
    # chunks with compatible exclusives, or make the compilation tasks do their own ivy resolves
    # for each batch of targets they're asked to compile.

    exclusives = Group._get_exclusives_product(context)

    sorted_excl_group_keys = exclusives.get_ordered_group_keys()
    all_chunks = []

    for excl_group_key in sorted_excl_group_keys:
      # TODO(John Sirois): coalescing should be made available in another spot, InternalTarget is jvm
      # specific, and all we care is that the Targets have dependencies defined

      chunk_targets = exclusives.get_targets_for_group_key(excl_group_key)
      # need to extract the targets for this chunk that are internal.
      ## TODO(markcc): right here, we're using "context.targets", which doesn't respect any of the
      ## exclusives rubbish going on around here.
      #coalesced = InternalTarget.coalesce_targets(context.targets(is_internal), discriminator)
      coalesced = InternalTarget.coalesce_targets(filter(is_internal, chunk_targets), discriminator)
      coalesced = list(reversed(coalesced))

      def not_internal(target):
        return not is_internal(target)
      # got targets that aren't internal.
      #rest = OrderedSet(context.targets(not_internal))
      rest = OrderedSet(filter(not_internal, chunk_targets))


      chunks = [rest] if rest else []
      flavor = None
      chunk_start = 0
      for i, target in enumerate(coalesced):
        target_flavor = discriminator(target)
        if target_flavor != flavor and i > chunk_start:
          chunks.append(OrderedSet(coalesced[chunk_start:i]))
          chunk_start = i
        flavor = target_flavor
      if chunk_start < len(coalesced):
        chunks.append(OrderedSet(coalesced[chunk_start:]))
      all_chunks += chunks

    context.log.debug('::: created chunks(%d)' % len(all_chunks))
    for i, chunk in enumerate(all_chunks):
      context.log.debug('  chunk(%d):\n\t%s' % (i, '\n\t'.join(sorted(map(str, chunk)))))

    return all_chunks
Пример #20
0
  def exported_targets(self):
    candidates = set()
    if self.transitive:
      candidates.update(self.context.targets())
    else:
      candidates.update(self.context.target_roots)

      def get_synthetic(lang, target):
        mappings = self.context.products.get(lang).get(target)
        if mappings:
          for generated in mappings.itervalues():
            for synthetic in generated:
              yield synthetic

      # Handle the case where a code gen target is in the listed roots and the thus the publishable
      # target is a synthetic twin generated by a code gen task upstream.
      for candidate in self.context.target_roots:
        candidates.update(get_synthetic('java', candidate))
        candidates.update(get_synthetic('scala', candidate))

    def exportable(tgt):
      return tgt in candidates and tgt.is_exported

    return OrderedSet(filter(exportable,
                             reversed(InternalTarget.sort_targets(filter(exportable, candidates)))))
Пример #21
0
    def __init__(self, root_dir, parser, argv):
        Command.__init__(self, root_dir, parser, argv)

        if not self.args:
            self.error("A spec argument is required")

        try:
            specs_end = self.args.index('--')
            if len(self.args) > specs_end:
                self.build_args = self.args.__getslice__(
                    specs_end + 1,
                    len(self.args) + 1)
            else:
                self.build_args = []
        except ValueError:
            specs_end = 1
            self.build_args = self.args[1:] if len(self.args) > 1 else []

        self.targets = OrderedSet()
        for spec in self.args.__getslice__(0, specs_end):
            try:
                address = Address.parse(root_dir, spec)
            except:
                self.error("Problem parsing spec %s: %s" %
                           (spec, traceback.format_exc()))

            try:
                target = Target.get(address)
            except:
                self.error("Problem parsing BUILD target %s: %s" %
                           (address, traceback.format_exc()))

            try:
                InternalTarget.check_cycles(target)
            except CycleException as e:
                self.error("Target contains an internal dependency cycle: %s" %
                           e)

            if not target:
                self.error("Target %s does not exist" % address)
            if not target.address.is_meta:
                target.address.is_meta = self.options.is_meta or address.is_meta
            self.targets.add(target)

        self.is_ide = self.options.is_ide
        self.ide_transitivity = self.options.ide_transitivity
Пример #22
0
  def __init__(self, root_dir, parser, argv):
    Command.__init__(self, root_dir, parser, argv)

    if not self.args:
      self.error("A spec argument is required")

    try:
      specs_end = self.args.index('--')
      if len(self.args) > specs_end:
        self.build_args = self.args.__getslice__(specs_end + 1, len(self.args) + 1)
      else:
        self.build_args = []
    except ValueError:
      specs_end = 1
      self.build_args = self.args[1:] if len(self.args) > 1 else []

    self.targets = OrderedSet()
    for spec in self.args.__getslice__(0, specs_end):
      try:
        address = Address.parse(root_dir, spec)
      except:
        self.error("Problem parsing spec %s: %s" % (spec, traceback.format_exc()))

      try:
        target = Target.get(address)
      except:
        self.error("Problem parsing BUILD target %s: %s" % (address, traceback.format_exc()))

      try:
        InternalTarget.check_cycles(target)
      except CycleException as e:
        self.error("Target contains an internal dependency cycle: %s" % e)

      if not target:
        self.error("Target %s does not exist" % address)
      if not target.address.is_meta:
        target.address.is_meta = self.options.is_meta or address.is_meta
      self.targets.add(target)

    self.is_ide = self.options.is_ide
    self.ide_transitivity = self.options.ide_transitivity
 def _compute_transitive_deps_by_target(self):
   # Sort from least to most dependent.
   sorted_targets = reversed(InternalTarget.sort_targets(self._context.targets()))
   transitive_deps_by_target = defaultdict(set)
   # Iterate in dep order, to accumulate the transitive deps for each target.
   for target in sorted_targets:
     transitive_deps = set()
     if hasattr(target, 'dependencies'):
       for dep in target.dependencies:
         transitive_deps.update(transitive_deps_by_target.get(dep, []))
         transitive_deps.add(dep)
       transitive_deps_by_target[target] = transitive_deps
   return transitive_deps_by_target
Пример #24
0
    def exported_targets(self):
        candidates = set(self.context.targets() if self.transitive else self.
                         context.target_roots)

        def exportable(target):
            return target in candidates and is_exported(
                target) and is_internal(target)

        return OrderedSet(
            filter(
                exportable,
                reversed(
                    InternalTarget.sort_targets(filter(exportable,
                                                       candidates)))))
Пример #25
0
 def _compute_transitive_deps_by_target(self):
   """Map from target to all the targets it depends on, transitively."""
   # Sort from least to most dependent.
   sorted_targets = reversed(InternalTarget.sort_targets(self._context.targets()))
   transitive_deps_by_target = defaultdict(set)
   # Iterate in dep order, to accumulate the transitive deps for each target.
   for target in sorted_targets:
     transitive_deps = set()
     if hasattr(target, 'dependencies'):
       for dep in target.dependencies:
         transitive_deps.update(transitive_deps_by_target.get(dep, []))
         transitive_deps.add(dep)
       transitive_deps_by_target[target] = transitive_deps
   return transitive_deps_by_target
Пример #26
0
 def __init__(self, name, dependencies=None, num_sources=0):
   with ParseContext.temp():
     InternalTarget.__init__(self, name, dependencies)
     TargetWithSources.__init__(self, name)
   self.num_sources = num_sources
Пример #27
0
 def exported_targets(self):
   candidates = set(self.context.targets() if self.transitive else self.context.target_roots)
   def exportable(target):
     return target in candidates and is_exported(target) and is_internal(target)
   return OrderedSet(filter(exportable,
                            reversed(InternalTarget.sort_targets(filter(exportable, candidates)))))
Пример #28
0
    def _create_chunks(context, goals):
        def discriminator(target):
            for i, goal in enumerate(goals):
                if goal.group.predicate(target):
                    return i
            return 'other'

        # First, divide the set of all targets to be built into compatible chunks, based
        # on their declared exclusives. Then, for each chunk of compatible exclusives, do
        # further subchunking. At the end, we'll have a list of chunks to be built,
        # which will go through the chunks of each exclusives-compatible group separately.

        # TODO(markcc); chunks with incompatible exclusives require separate ivy resolves.
        # Either interleave the ivy task in this group so that it runs once for each batch of
        # chunks with compatible exclusives, or make the compilation tasks do their own ivy resolves
        # for each batch of targets they're asked to compile.

        exclusives = Group._get_exclusives_product(context)

        sorted_excl_group_keys = exclusives.get_ordered_group_keys()
        all_chunks = []

        for excl_group_key in sorted_excl_group_keys:
            # TODO(John Sirois): coalescing should be made available in another spot, InternalTarget is jvm
            # specific, and all we care is that the Targets have dependencies defined

            chunk_targets = exclusives.get_targets_for_group_key(
                excl_group_key)
            # need to extract the targets for this chunk that are internal.
            ## TODO(markcc): right here, we're using "context.targets", which doesn't respect any of the
            ## exclusives rubbish going on around here.
            #coalesced = InternalTarget.coalesce_targets(context.targets(is_internal), discriminator)
            coalesced = InternalTarget.coalesce_targets(
                filter(is_internal, chunk_targets), discriminator)
            coalesced = list(reversed(coalesced))

            def not_internal(target):
                return not is_internal(target)

            # got targets that aren't internal.
            #rest = OrderedSet(context.targets(not_internal))
            rest = OrderedSet(filter(not_internal, chunk_targets))

            chunks = [rest] if rest else []
            flavor = None
            chunk_start = 0
            for i, target in enumerate(coalesced):
                target_flavor = discriminator(target)
                if target_flavor != flavor and i > chunk_start:
                    chunks.append(OrderedSet(coalesced[chunk_start:i]))
                    chunk_start = i
                flavor = target_flavor
            if chunk_start < len(coalesced):
                chunks.append(OrderedSet(coalesced[chunk_start:]))
            all_chunks += chunks

        context.log.debug('::: created chunks(%d)' % len(all_chunks))
        for i, chunk in enumerate(all_chunks):
            flavor = discriminator(iter(chunk).next())
            context.log.debug(
                '  chunk(%d) [flavor=%s]:\n\t%s' %
                (i, flavor, '\n\t'.join(sorted(map(str, chunk)))))

        return all_chunks
Пример #29
0
 def __init__(self, name, dependencies=None, num_sources=0):
     with ParseContext.temp():
         InternalTarget.__init__(self, name, dependencies)
         TargetWithSources.__init__(self, name)
     self.num_sources = num_sources
Пример #30
0
def extract_target(java_targets, name=None):
    """Extracts a minimal set of linked targets from the given target's internal transitive dependency
  set.  The root target in the extracted target set is returned.  The algorithm does a topological
  sort of the internal targets and then tries to coalesce targets of a given type.  Any target with
  a custom ant build xml will be excluded from the coalescing."""

    # TODO(John Sirois): this is broken - representative_target is not necessarily representative
    representative_target = list(java_targets)[0]

    meta_target_base_name = "fast-%s" % (name if name else
                                         representative_target.name)
    provides = None
    deployjar = hasattr(representative_target,
                        'deployjar') and representative_target.deployjar
    buildflags = representative_target.buildflags

    def discriminator(tgt):
        # Chunk up our targets by (type, src base) - the javac task in the ant build relies upon a
        # single srcdir that points to the root of a package tree to ensure differential compilation
        # works.
        return type(tgt), tgt.target_base

    def create_target(category, target_name, target_index, targets):
        def name(name):
            return "%s-%s-%d" % (target_name, name, target_index)

        # TODO(John Sirois): JavaLibrary and ScalaLibrary can float here between src/ and tests/ - add
        # ant build support to allow the same treatment for JavaThriftLibrary and JavaProtobufLibrary
        # so that tests can house test IDL in tests/
        target_type, base = category
        if target_type == JavaProtobufLibrary:
            return JavaProtobufLibrary._aggregate(name('protobuf'), provides,
                                                  buildflags, targets)
        elif target_type == JavaThriftLibrary:
            return JavaThriftLibrary._aggregate(name('thrift'), provides,
                                                buildflags, targets)
        elif target_type == AnnotationProcessor:
            return AnnotationProcessor._aggregate(name('apt'), provides,
                                                  targets)
        elif target_type == JavaLibrary:
            return JavaLibrary._aggregate(name('java'), provides, deployjar,
                                          buildflags, targets, base)
        elif target_type == ScalaLibrary:
            return ScalaLibrary._aggregate(name('scala'), provides, deployjar,
                                           buildflags, targets, base)
        elif target_type == JavaTests:
            return JavaTests._aggregate(name('java-tests'), buildflags,
                                        targets)
        elif target_type == ScalaTests:
            return ScalaTests._aggregate(name('scala-tests'), buildflags,
                                         targets)
        else:
            raise Exception("Cannot aggregate targets of type: %s" %
                            target_type)

    # TODO(John Sirois): support a flag that selects conflict resolution policy - this currently
    # happens to mirror the ivy policy we use
    def resolve_conflicts(target):
        dependencies = {}
        for dependency in target.resolved_dependencies:
            for jar in dependency._as_jar_dependencies():
                key = jar.org, jar.name
                previous = dependencies.get(key, jar)
                if jar.rev >= previous.rev:
                    if jar != previous:
                        print "WARNING: replacing %s with %s for %s" % (
                            previous, jar, target._id)
                        target.resolved_dependencies.remove(previous)
                        target.jar_dependencies.remove(previous)
                    dependencies[key] = jar
        return target

    # chunk up our targets by type & custom build xml
    coalesced = InternalTarget.coalesce_targets(java_targets, discriminator)
    coalesced = list(reversed(coalesced))

    start_type = discriminator(coalesced[0])
    start = 0
    descriptors = []

    for current in range(0, len(coalesced)):
        current_target = coalesced[current]
        current_type = discriminator(current_target)

        if current_target.custom_antxml_path:
            if start < current:
                # if we have a type chunk to our left, record it
                descriptors.append((start_type, coalesced[start:current]))

            # record a chunk containing just the target that has the custom build xml to be conservative
            descriptors.append((current_type, [current_target]))
            start = current + 1
            if current < (len(coalesced) - 1):
                start_type = discriminator(coalesced[start])

        elif start_type != current_type:
            # record the type chunk we just left
            descriptors.append((start_type, coalesced[start:current]))
            start = current
            start_type = current_type

    if start < len(coalesced):
        # record the tail chunk
        descriptors.append((start_type, coalesced[start:]))

    # build meta targets aggregated from the chunks and keep track of which targets end up in which
    # meta targets
    meta_targets_by_target_id = dict()
    targets_by_meta_target = []
    for (ttype, targets), index in zip(descriptors,
                                       reversed(range(0, len(descriptors)))):
        meta_target = resolve_conflicts(
            create_target(ttype, meta_target_base_name, index, targets))
        targets_by_meta_target.append((meta_target, targets))
        for target in targets:
            meta_targets_by_target_id[target._id] = meta_target

    # calculate the other meta-targets (if any) each meta-target depends on
    extra_targets_by_meta_target = []
    for meta_target, targets in targets_by_meta_target:
        meta_deps = set()
        custom_antxml_path = None
        for target in targets:
            if target.custom_antxml_path:
                custom_antxml_path = target.custom_antxml_path
            for dep in target.resolved_dependencies:
                if is_jvm(dep):
                    meta = meta_targets_by_target_id[dep._id]
                    if meta != meta_target:
                        meta_deps.add(meta)
        extra_targets_by_meta_target.append(
            (meta_target, meta_deps, custom_antxml_path))

    def lift_excludes(meta_target):
        excludes = set()

        def lift(target):
            if target.excludes:
                excludes.update(target.excludes)
            for jar_dep in target.jar_dependencies:
                excludes.update(jar_dep.excludes)
            for internal_dep in target.internal_dependencies:
                lift(internal_dep)

        lift(meta_target)
        return excludes

    # link in the extra inter-meta deps
    meta_targets = []
    for meta_target, extra_deps, custom_antxml_path in extra_targets_by_meta_target:
        meta_targets.append(meta_target)
        meta_target.update_dependencies(extra_deps)
        meta_target.excludes = lift_excludes(meta_target)
        meta_target.custom_antxml_path = custom_antxml_path

    sorted_meta_targets = InternalTarget.sort_targets(meta_targets)

    def prune_metas(target):
        if sorted_meta_targets:
            try:
                sorted_meta_targets.remove(target)
            except ValueError:
                # we've already removed target in the current walk
                pass

    # link any disconnected meta_target graphs so we can return 1 root target
    root = None
    while sorted_meta_targets:
        new_root = sorted_meta_targets[0]
        new_root.walk(prune_metas, is_jvm)
        if root:
            new_root.update_dependencies([root])
        root = new_root

    return root
Пример #31
0
def extract_target(java_targets, name = None):
  """Extracts a minimal set of linked targets from the given target's internal transitive dependency
  set.  The root target in the extracted target set is returned.  The algorithm does a topological
  sort of the internal targets and then tries to coalesce targets of a given type.  Any target with
  a custom ant build xml will be excluded from the coalescing."""

  # TODO(John Sirois): this is broken - representative_target is not necessarily representative
  representative_target = list(java_targets)[0]

  meta_target_base_name = "fast-%s" % (name if name else representative_target.name)
  provides = None
  deployjar = hasattr(representative_target, 'deployjar') and representative_target.deployjar
  buildflags = representative_target.buildflags

  def discriminator(tgt):
    # Chunk up our targets by (type, src base) - the javac task in the ant build relies upon a
    # single srcdir that points to the root of a package tree to ensure differential compilation
    # works.
    return type(tgt), tgt.target_base

  def create_target(category, target_name, target_index, targets):
    def name(name):
      return "%s-%s-%d" % (target_name, name, target_index)

    # TODO(John Sirois): JavaLibrary and ScalaLibrary can float here between src/ and tests/ - add
    # ant build support to allow the same treatment for JavaThriftLibrary and JavaProtobufLibrary
    # so that tests can house test IDL in tests/
    target_type, base = category
    if target_type == JavaProtobufLibrary:
      return JavaProtobufLibrary._aggregate(name('protobuf'), provides, buildflags, targets)
    elif target_type == JavaThriftLibrary:
      return JavaThriftLibrary._aggregate(name('thrift'), provides, buildflags, targets)
    elif target_type == AnnotationProcessor:
      return AnnotationProcessor._aggregate(name('apt'), provides, targets)
    elif target_type == JavaLibrary:
      return JavaLibrary._aggregate(name('java'), provides, deployjar, buildflags, targets, base)
    elif target_type == ScalaLibrary:
      return ScalaLibrary._aggregate(name('scala'), provides, deployjar, buildflags, targets, base)
    elif target_type == JavaTests:
      return JavaTests._aggregate(name('java-tests'), buildflags, targets)
    elif target_type == ScalaTests:
      return ScalaTests._aggregate(name('scala-tests'), buildflags, targets)
    else:
      raise Exception("Cannot aggregate targets of type: %s" % target_type)

  # TODO(John Sirois): support a flag that selects conflict resolution policy - this currently
  # happens to mirror the ivy policy we use
  def resolve_conflicts(target):
    dependencies = {}
    for dependency in target.resolved_dependencies:
      for jar in dependency._as_jar_dependencies():
        key = jar.org, jar.name
        previous = dependencies.get(key, jar)
        if jar.rev >= previous.rev:
          if jar != previous:
            print "WARNING: replacing %s with %s for %s" % (previous, jar, target._id)
            target.resolved_dependencies.remove(previous)
            target.jar_dependencies.remove(previous)
          dependencies[key] = jar
    return target

  # chunk up our targets by type & custom build xml
  coalesced = InternalTarget.coalesce_targets(java_targets, discriminator)
  coalesced = list(reversed(coalesced))

  start_type = discriminator(coalesced[0])
  start = 0
  descriptors = []

  for current in range(0, len(coalesced)):
    current_target = coalesced[current]
    current_type = discriminator(current_target)

    if current_target.custom_antxml_path:
      if start < current:
        # if we have a type chunk to our left, record it
        descriptors.append((start_type, coalesced[start:current]))

      # record a chunk containing just the target that has the custom build xml to be conservative
      descriptors.append((current_type, [current_target]))
      start = current + 1
      if current < (len(coalesced) - 1):
        start_type = discriminator(coalesced[start])

    elif start_type != current_type:
      # record the type chunk we just left
      descriptors.append((start_type, coalesced[start:current]))
      start = current
      start_type = current_type

  if start < len(coalesced):
    # record the tail chunk
    descriptors.append((start_type, coalesced[start:]))

  # build meta targets aggregated from the chunks and keep track of which targets end up in which
  # meta targets
  meta_targets_by_target_id = dict()
  targets_by_meta_target = []
  for (ttype, targets), index in zip(descriptors, reversed(range(0, len(descriptors)))):
    meta_target = resolve_conflicts(create_target(ttype, meta_target_base_name, index, targets))
    targets_by_meta_target.append((meta_target, targets))
    for target in targets:
      meta_targets_by_target_id[target._id] = meta_target

  # calculate the other meta-targets (if any) each meta-target depends on
  extra_targets_by_meta_target = []
  for meta_target, targets in targets_by_meta_target:
    meta_deps = set()
    custom_antxml_path = None
    for target in targets:
      if target.custom_antxml_path:
        custom_antxml_path = target.custom_antxml_path
      for dep in target.resolved_dependencies:
        if is_jvm(dep):
          meta = meta_targets_by_target_id[dep._id]
          if meta != meta_target:
            meta_deps.add(meta)
    extra_targets_by_meta_target.append((meta_target, meta_deps, custom_antxml_path))

  def lift_excludes(meta_target):
    excludes = set()
    def lift(target):
      if target.excludes:
        excludes.update(target.excludes)
      for jar_dep in target.jar_dependencies:
        excludes.update(jar_dep.excludes)
      for internal_dep in target.internal_dependencies:
        lift(internal_dep)
    lift(meta_target)
    return excludes

  # link in the extra inter-meta deps
  meta_targets = []
  for meta_target, extra_deps, custom_antxml_path in extra_targets_by_meta_target:
    meta_targets.append(meta_target)
    meta_target.update_dependencies(extra_deps)
    meta_target.excludes = lift_excludes(meta_target)
    meta_target.custom_antxml_path = custom_antxml_path

  sorted_meta_targets = InternalTarget.sort_targets(meta_targets)
  def prune_metas(target):
    if sorted_meta_targets:
      try:
        sorted_meta_targets.remove(target)
      except ValueError:
        # we've already removed target in the current walk
        pass

  # link any disconnected meta_target graphs so we can return 1 root target
  root = None
  while sorted_meta_targets:
    new_root = sorted_meta_targets[0]
    new_root.walk(prune_metas, is_jvm)
    if root:
      new_root.update_dependencies([root])
    root = new_root

  return root