def get_all_deps():
all_deps = OrderedSet([
Pants('3rdparty:scala-library').resolve(),
])
if dependencies:
all_deps.update(dependencies)
return all_deps
def _run_tests(self, targets, args):
testargs = OrderedSet([ 'py.test' ])
if args:
testargs.update(args)
def add_tests(template_data):
if template_data.sources:
basedir = template_data.template_base
testargs.update(os.path.join(basedir, test) for test in template_data.sources)
if template_data.dependencies:
for dependency in template_data.dependencies:
for dep in dependency.resolve():
add_tests(dep._create_template_data())
for target in targets:
template_data = target._create_template_data()
add_tests(template_data)
print 'PythonBuilder executing (PYTHONPATH="%s") %s' % (
os.environ['PYTHONPATH'],
' '.join(testargs)
)
return subprocess.call(testargs, cwd = self.root_dir)
def _aggregate(cls, name, provides, deployjar, buildflags, scala_libs):
all_deps = OrderedSet()
all_excludes = OrderedSet()
all_sources = []
all_java_sources = []
all_resources = []
all_binary_resources = []
for scala_lib in scala_libs:
if scala_lib.resolved_dependencies:
all_deps.update(dep for dep in scala_lib.jar_dependencies if dep.rev is not None)
if scala_lib.excludes:
all_excludes.update(scala_lib.excludes)
if scala_lib.sources:
all_sources.extend(scala_lib.sources)
if scala_lib.java_sources:
all_java_sources.extend(scala_lib.java_sources)
if scala_lib.resources:
all_resources.extend(scala_lib.resources)
if scala_lib.binary_resources:
all_binary_resources.extend(scala_lib.binary_resources)
return ScalaLibrary(name,
all_sources,
java_sources = all_java_sources,
provides = provides,
dependencies = all_deps,
excludes = all_excludes,
resources = all_resources,
binary_resources = all_binary_resources,
deployjar = deployjar,
buildflags = buildflags,
is_meta = True)
def scan_addresses(root_dir, base_path=None):
"""Parses all targets available in BUILD files under base_path and returns their addresses. If no
base_path is specified, root_dir is assumed to be the base_path"""
addresses = OrderedSet()
for buildfile in BuildFile.scan_buildfiles(root_dir, base_path):
addresses.update(Target.get_all_addresses(buildfile))
return addresses
def scan_addresses(root_dir, base_path = None):
"""Parses all targets available in BUILD files under base_path and returns their addresses. If no
base_path is specified, root_dir is assumed to be the base_path"""
addresses = OrderedSet()
for buildfile in BuildFile.scan_buildfiles(root_dir, base_path):
addresses.update(Target.get_all_addresses(buildfile))
return addresses
def get_all_deps():
all_deps = OrderedSet([
Pants('src/scala/com/twitter/common/testing:explicit-specs-runner').resolve(),
Pants('3rdparty:scala-library').resolve(),
])
if dependencies:
all_deps.update(dependencies)
return all_deps
def get_all_deps():
all_deps = OrderedSet([
Pants('3rdparty:commons-lang').resolve(),
JarDependency(org = 'org.apache.thrift',
name = 'libthrift',
rev = '${thrift.library.version}'),
Pants('3rdparty:slf4j-api').resolve(),
])
if dependencies:
all_deps.update(dependencies)
return all_deps
def _aggregate(cls, name, buildflags, scala_tests):
all_deps = OrderedSet()
all_excludes = OrderedSet()
all_sources = []
for scala_test in scala_tests:
if scala_test.resolved_dependencies:
all_deps.update(dep for dep in scala_test.jar_dependencies if dep.rev is not None)
if scala_test.excludes:
all_excludes.update(scala_test.excludes)
if scala_test.sources:
all_sources.extend(scala_test.sources)
return ScalaTests(name,
all_sources,
dependencies = all_deps,
buildflags = buildflags,
is_meta = True)
def _aggregate(cls, name, provides, buildflags, java_proto_libs):
all_sources = []
all_deps = OrderedSet()
all_excludes = OrderedSet()
for java_proto_lib in java_proto_libs:
if java_proto_lib.sources:
all_sources.extend(java_proto_lib.sources)
if java_proto_lib.resolved_dependencies:
all_deps.update(dep for dep in java_proto_lib.jar_dependencies if dep.rev is not None)
if java_proto_lib.excludes:
all_excludes.update(java_proto_lib.excludes)
return JavaProtobufLibrary(name,
all_sources,
provides = provides,
dependencies = all_deps,
excludes = all_excludes,
buildflags = buildflags,
is_meta = True)
class JarLibrary(Target):
"""Serves as a proxy for one or more JarDependencies."""
def __init__(self, name, *dependencies):
"""name: The name of this module target, addressable via pants via the portion of the spec
following the colon
dependencies: one or more JarDependencies this JarLibrary bundles or Pants pointing to other
JarLibraries"""
assert len(dependencies) > 0, "At least one dependency must be specified"
Target.__init__(self, name, False)
self.jar_dependencies = OrderedSet()
for dependency in dependencies:
self.jar_dependencies.update((dependency.resolve())._as_jar_dependencies())
def _as_jar_dependencies(self):
for jar in self.jar_dependencies:
yield jar
def extract_target(java_target, is_transitive):
meta_target = bang.extract_target(java_target)
internal_deps, jar_deps = _extract_target(meta_target, is_transitive)
# TODO(John Sirois): make an empty source set work in ant/compile.xml
sources = [ '__no_source__' ]
all_deps = OrderedSet()
all_deps.update(internal_deps)
all_deps.update(jar_deps)
return JavaLibrary('ide',
sources,
provides = None,
dependencies = all_deps,
excludes = meta_target.excludes,
resources = None,
binary_resources = None,
deployjar = False,
buildflags = None,
is_meta = True)
def extract_target(java_targets, is_transitive, name=None):
meta_target = bang.extract_target(java_targets, name)
internal_deps, jar_deps = _extract_target(meta_target, is_transitive)
# TODO(John Sirois): make an empty source set work in ant/compile.xml
sources = ['__no_source__']
all_deps = OrderedSet()
all_deps.update(internal_deps)
all_deps.update(jar_deps)
return JavaLibrary('ide',
sources,
provides=None,
dependencies=all_deps,
excludes=meta_target.excludes,
resources=None,
binary_resources=None,
deployjar=False,
buildflags=None,
is_meta=True)
class InternalTarget(Target):
"""A baseclass for targets that support an optional dependency set."""
@classmethod
def check_cycles(cls, internal_target):
"""Validates the given InternalTarget has no circular dependencies. Raises CycleException if
it does."""
dep_stack = OrderedSet()
def descend(internal_dep):
if internal_dep in dep_stack:
raise CycleException(dep_stack, internal_dep)
if hasattr(internal_dep, 'internal_dependencies'):
dep_stack.add(internal_dep)
for dep in internal_dep.internal_dependencies:
descend(dep)
dep_stack.remove(internal_dep)
descend(internal_target)
@classmethod
def sort_targets(cls, internal_targets):
"""Returns a list of targets that internal_targets depend on sorted from most dependent to
least."""
roots = OrderedSet()
inverted_deps = collections.defaultdict(OrderedSet) # target -> dependent targets
visited = set()
def invert(target):
if target not in visited:
visited.add(target)
if target.internal_dependencies:
for internal_dependency in target.internal_dependencies:
if isinstance(internal_dependency, InternalTarget):
inverted_deps[internal_dependency].add(target)
invert(internal_dependency)
else:
roots.add(target)
for internal_target in internal_targets:
invert(internal_target)
sorted = []
visited.clear()
def topological_sort(target):
if target not in visited:
visited.add(target)
if target in inverted_deps:
for dep in inverted_deps[target]:
topological_sort(dep)
sorted.append(target)
for root in roots:
topological_sort(root)
return sorted
@classmethod
def coalesce_targets(cls, internal_targets):
"""Returns a list of targets internal_targets depend on sorted from most dependent to least and
grouped where possible by target type."""
sorted_targets = InternalTarget.sort_targets(internal_targets)
# can do no better for any of these:
# []
# [a]
# [a,b]
if len(sorted_targets) <= 2:
return sorted_targets
# For these, we'd like to coalesce if possible, like:
# [a,b,a,c,a,c] -> [a,a,a,b,c,c]
# adopt a quadratic worst case solution, when we find a type change edge, scan forward for
# the opposite edge and then try to swap dependency pairs to move the type back left to its
# grouping. If the leftwards migration fails due to a dependency constraint, we just stop
# and move on leaving "type islands".
current_type = None
# main scan left to right no backtracking
for i in range(len(sorted_targets) - 1):
current_target = sorted_targets[i]
if current_type != type(current_target):
scanned_back = False
# scan ahead for next type match
for j in range(i + 1, len(sorted_targets)):
look_ahead_target = sorted_targets[j]
if current_type == type(look_ahead_target):
scanned_back = True
# swap this guy as far back as we can
for k in range(j, i, -1):
previous_target = sorted_targets[k - 1]
mismatching_types = current_type != type(previous_target)
not_a_dependency = look_ahead_target not in previous_target.internal_dependencies
if mismatching_types and not_a_dependency:
sorted_targets[k] = sorted_targets[k - 1]
sorted_targets[k - 1] = look_ahead_target
else:
break # out of k
break # out of j
if not scanned_back: # done with coalescing the current type, move on to next
current_type = type(current_target)
return sorted_targets
def sort(self):
"""Returns a list of targets this target depends on sorted from most dependent to least."""
return InternalTarget.sort_targets([ self ])
def coalesce(self):
"""Returns a list of targets this target depends on sorted from most dependent to least and
grouped where possible by target type."""
return InternalTarget.coalesce_targets([ self ])
def __init__(self, name, dependencies, is_meta):
Target.__init__(self, name, is_meta)
self.resolved_dependencies = OrderedSet()
self.internal_dependencies = OrderedSet()
self.jar_dependencies = OrderedSet()
self.update_dependencies(dependencies)
def update_dependencies(self, dependencies):
if dependencies:
for dependency in dependencies:
for resolved_dependency in dependency.resolve():
self.resolved_dependencies.add(resolved_dependency)
if isinstance(resolved_dependency, InternalTarget):
self.internal_dependencies.add(resolved_dependency)
self.jar_dependencies.update(resolved_dependency._as_jar_dependencies())
def _walk(self, walked, work, predicate = None):
Target._walk(self, walked, work, predicate)
for dep in self.resolved_dependencies:
if isinstance(dep, Target) and not dep in walked:
walked.add(dep)
if not predicate or predicate(dep):
additional_targets = work(dep)
dep._walk(walked, work, predicate)
if additional_targets:
for additional_target in additional_targets:
additional_target._walk(walked, work, predicate)
def get_all_deps():
all_deps = OrderedSet()
all_deps.update(Pants('3rdparty:junit').resolve())
if dependencies:
all_deps.update(dependencies)
return all_deps
class InternalTarget(Target):
"""A baseclass for targets that support an optional dependency set."""
def sort(self):
"""Returns a list of targets this target depends on sorted from most dependent to least."""
roots = dict() # address -> root target
inverted_deps = collections.defaultdict(OrderedSet) # address -> dependent targets
visited = set() # addresses
def invert(target):
if target.address not in visited:
visited.add(target.address)
if target.internal_dependencies:
for internal_dependency in target.internal_dependencies:
if isinstance(internal_dependency, InternalTarget):
inverted_deps[internal_dependency.address].add(target)
invert(internal_dependency)
else:
roots[target.address] = target
invert(self)
sorted = []
visited.clear()
def topological_sort(target):
if target.address not in visited:
visited.add(target.address)
if target.address in inverted_deps:
for dep in inverted_deps[target.address]:
topological_sort(dep)
sorted.append(target)
for root in roots.values():
topological_sort(root)
return sorted
def coalesce(self):
"""Returns a list of targets this target depends on sorted from most dependent to least and
grouped where possible by target type."""
sorted_targets = self.sort()
# can do no better for any of these:
# []
# [a]
# [a,b]
if len(sorted_targets) <= 2:
return sorted_targets
# For these, we'd like to coalesce if possible, like:
# [a,b,a,c,a,c] -> [a,a,a,b,c,c]
# adopt a quadratic worst case solution, when we find a type change edge, scan forward for
# the opposite edge and then try to swap dependency pairs to move the type back left to its
# grouping. If the leftwards migration fails due to a dependency constraint, we just stop
# and move on leaving "type islands".
current_type = type(self)
# main scan left to right no backtracking
for i in range(len(sorted_targets) - 1):
current_target = sorted_targets[i]
if current_type != type(current_target):
scanned_back = False
# scan ahead for next type match
for j in range(i + 1, len(sorted_targets)):
look_ahead_target = sorted_targets[j]
if current_type == type(look_ahead_target):
scanned_back = True
# swap this guy as far back as we can
for k in range(j, i, -1):
previous_target = sorted_targets[k - 1]
mismatching_types = current_type != type(previous_target)
not_a_dependency = look_ahead_target not in previous_target.internal_dependencies
if mismatching_types and not_a_dependency:
sorted_targets[k] = sorted_targets[k - 1]
sorted_targets[k - 1] = look_ahead_target
else:
break # out of k
break # out of j
if not scanned_back: # done with coalescing the current type, move on to next
current_type = type(current_target)
return sorted_targets
def __init__(self, name, dependencies, is_meta):
Target.__init__(self, name, is_meta)
self.resolved_dependencies = OrderedSet()
self.internal_dependencies = OrderedSet()
self.jar_dependencies = OrderedSet()
if dependencies:
for dependency in dependencies:
resolved_dependency = dependency.resolve()
self.resolved_dependencies.add(resolved_dependency)
if isinstance(resolved_dependency, InternalTarget):
self.internal_dependencies.add(resolved_dependency)
self.jar_dependencies.update(resolved_dependency._as_jar_dependencies())
def configure(self):
"""Configures this project's source sets returning the full set of targets the project is
comprised of. The full set can be larger than the initial set of targets when any of the
initial targets only has partial ownership of its parent directory source set."""
analyzed = OrderedSet()
targeted = set()
def accept_target(target):
return has_sources(target) and not target.is_codegen
def configure_source_sets(relative_base, sources, is_test):
absolute_base = os.path.join(self.root_dir, relative_base)
paths = set([os.path.dirname(source) for source in sources])
for path in paths:
absolute_path = os.path.join(absolute_base, path)
if absolute_path not in targeted:
targeted.add(absolute_path)
self.sources.append(
SourceSet(self.root_dir, relative_base, path, is_test))
def configure_target(target):
if target not in analyzed:
analyzed.add(target)
self.has_scala = self.has_scala or is_scala(target)
if isinstance(target, JavaLibrary) or isinstance(
target, ScalaLibrary):
# TODO(John Sirois): this does not handle test resources, make test resources 1st class
# in ant build and punch this through to pants model
resources = set()
if target.resources:
resources.update(target.resources)
if target.binary_resources:
resources.update(target.binary_resources)
if resources:
self.resource_extensions.update(
Project.extract_resource_extensions(resources))
configure_source_sets(RESOURCES_BASE_DIR,
resources,
is_test=False)
if target.sources:
test = is_test(target)
self.has_tests = self.has_tests or test
configure_source_sets(target.target_base,
target.sources,
is_test=test)
siblings = Target.get_all_addresses(target.address.buildfile)
return filter(
accept_target,
[Target.get(a) for a in siblings if a != target.address])
for target in self.targets:
target.walk(configure_target, predicate=accept_target)
for source_set in self.sources:
paths = set()
source_base = os.path.join(self.root_dir, source_set.source_base)
for root, dirs, _ in os.walk(
os.path.join(source_base, source_set.path)):
if dirs:
paths.update([os.path.join(root, dir) for dir in dirs])
unused_children = paths - targeted
if unused_children and paths != unused_children:
source_set.excludes.extend(
os.path.relpath(child, source_base)
for child in unused_children)
targets = OrderedSet()
for target in self.targets:
target.walk(lambda target: targets.add(target), has_sources)
targets.update(analyzed - targets)
return targets
def configure(self):
"""Configures this project's source sets returning the full set of targets the project is
comprised of. The full set can be larger than the initial set of targets when any of the
initial targets only has partial ownership of its parent directory source set."""
analyzed = OrderedSet()
targeted = set()
def accept_target(target):
return has_sources(target) and not target.is_codegen
def configure_source_sets(relative_base, sources, is_test):
absolute_base = os.path.join(self.root_dir, relative_base)
paths = set([ os.path.dirname(source) for source in sources])
for path in paths:
absolute_path = os.path.join(absolute_base, path)
if absolute_path not in targeted:
targeted.add(absolute_path)
self.sources.append(SourceSet(self.root_dir, relative_base, path, is_test))
def configure_target(target):
if target not in analyzed:
analyzed.add(target)
self.has_scala = self.has_scala or is_scala(target)
if isinstance(target, JavaLibrary) or isinstance(target, ScalaLibrary):
# TODO(John Sirois): this does not handle test resources, make test resources 1st class
# in ant build and punch this through to pants model
resources = set()
if target.resources:
resources.update(target.resources)
if target.binary_resources:
resources.update(target.binary_resources)
if resources:
self.resource_extensions.update(Project.extract_resource_extensions(resources))
configure_source_sets(RESOURCES_BASE_DIR, resources, is_test = False)
if target.sources:
test = is_test(target)
self.has_tests = self.has_tests or test
configure_source_sets(target.target_base, target.sources, is_test = test)
siblings = Target.get_all_addresses(target.address.buildfile)
return filter(accept_target, [ Target.get(a) for a in siblings if a != target.address ])
for target in self.targets:
target.walk(configure_target, predicate = accept_target)
for source_set in self.sources:
paths = set()
source_base = os.path.join(self.root_dir, source_set.source_base)
for root, dirs, _ in os.walk(os.path.join(source_base, source_set.path)):
if dirs:
paths.update([ os.path.join(root, dir) for dir in dirs ])
unused_children = paths - targeted
if unused_children and paths != unused_children:
source_set.excludes.extend(os.path.relpath(child, source_base) for child in unused_children)
targets = OrderedSet()
for target in self.targets:
target.walk(lambda target: targets.add(target), has_sources)
targets.update(analyzed - targets)
return targets
