def _create_doc_target(self):
all_sources = []
all_deps = OrderedSet()
for target in self.targets:
if not self.only_provides or is_exported(target):
for source in target.sources:
source_path = os.path.join(self.java_src_prefix, source)
if os.path.exists(source_path):
all_sources.append(source_path)
else:
print "skipping %s" % source_path
for jar_dep in target.jar_dependencies:
if jar_dep.rev:
all_deps.add(copy(jar_dep).intransitive())
def create_meta_target():
return JavaLibrary('pants.doc.deps',
all_sources,
provides = None,
dependencies = all_deps,
excludes = None,
resources = None,
binary_resources = None,
deployjar = False,
buildflags = None,
is_meta = True)
# TODO(John Sirois): Find a better way to do_in_context when we don't care about the context
return list(self.targets)[0].do_in_context(create_meta_target)
def _create_doc_target(self):
all_sources = []
all_deps = OrderedSet()
for target in self.targets:
if not self.only_provides or is_exported(target):
for source in target.sources:
source_path = os.path.join(self.java_src_prefix, source)
if os.path.exists(source_path):
all_sources.append(source_path)
else:
print "skipping %s" % source_path
for jar_dep in target.jar_dependencies:
if jar_dep.rev:
all_deps.add(copy(jar_dep).intransitive())
def create_meta_target():
return JavaLibrary('pants.doc.deps',
all_sources,
provides=None,
dependencies=all_deps,
excludes=None,
resources=None,
binary_resources=None,
deployjar=False,
buildflags=None,
is_meta=True)
# TODO(John Sirois): Find a better way to do_in_context when we don't care about the context
return list(self.targets)[0].do_in_context(create_meta_target)
def scan_buildfiles(cls, root_dir, base_path = None):
"""Looks for all BUILD files under base_path"""
buildfiles = OrderedSet()
for root, dirs, files in os.walk(base_path if base_path else root_dir):
for filename in files:
if BuildFile._is_buildfile_name(filename):
buildfile_relpath = os.path.relpath(os.path.join(root, filename), root_dir)
buildfiles.add(BuildFile(root_dir, buildfile_relpath))
return buildfiles
def _parse_buildfiles(self):
buildfiles = OrderedSet()
for spec in self.args:
try:
if self.options.is_directory_list:
for buildfile in BuildFile.scan_buildfiles(self.root_dir, spec):
buildfiles.add(buildfile)
else:
buildfiles.add(Address.parse(self.root_dir, spec).buildfile)
except:
self.error("Problem parsing spec %s: %s" % (spec, traceback.format_exc()))
return buildfiles
def _parse_buildfiles(self):
buildfiles = OrderedSet()
for spec in self.args:
try:
if self.options.is_directory_list:
for buildfile in BuildFile.scan_buildfiles(self.root_dir, spec):
buildfiles.add(buildfile)
else:
buildfiles.add(Address.parse(self.root_dir, spec).buildfile)
except:
self.error("Problem parsing spec %s: %s" % (spec, traceback.format_exc()))
return buildfiles
def _parse_addresses(self):
addresses = OrderedSet()
for spec in self.args:
try:
if self.options.is_directory_list:
for address in scan_addresses(self.root_dir, spec):
addresses.add(address)
else:
addresses.add(Address.parse(self.root_dir, spec))
except:
self.error("Problem parsing spec %s: %s" % (spec, traceback.format_exc()))
return addresses
def _parse_addresses(self):
addresses = OrderedSet()
for spec in self.args:
try:
if self.options.is_directory_list:
for address in scan_addresses(self.root_dir, spec):
addresses.add(address)
else:
addresses.add(Address.parse(self.root_dir, spec))
except:
self.error("Problem parsing spec %s: %s" %
(spec, traceback.format_exc()))
return addresses
def _parse_targets(self, root_dir):
targets = OrderedSet()
for spec in self.args:
try:
address = Address.parse(root_dir, spec)
except:
self.error("Problem parsing spec %s: %s" % (spec, traceback.format_exc()))
try:
target = Target.get(address)
except:
self.error("Problem parsing target %s: %s" % (address, traceback.format_exc()))
if not Doc._is_documentable(target):
self.error("Target: %s is not documentable" % address)
targets.add(target)
return targets
def _parse_targets(self, root_dir):
targets = OrderedSet()
for spec in self.args:
try:
address = Address.parse(root_dir, spec)
except:
self.error("Problem parsing spec %s: %s" % (spec, traceback.format_exc()))
try:
target = Target.get(address)
except:
self.error("Problem parsing target %s: %s" % (address, traceback.format_exc()))
if not Doc._is_documentable(target):
self.error("Target: %s is not documentable" % address)
targets.add(target)
return targets
def execute(self):
print "Build operating on targets: %s" % self.targets
jvm_targets = OrderedSet()
python_targets = OrderedSet()
for target in self.targets:
if is_jvm(target):
jvm_targets.add(target)
elif is_python(target):
python_targets.add(target)
else:
self.error("Cannot build target %s" % target)
if jvm_targets:
status = self._jvm_build(jvm_targets)
if status != 0:
return status
if python_targets:
status = self._python_build(python_targets)
return status
def execute(self):
print "Build operating on targets: %s" % self.targets
jvm_targets = OrderedSet()
python_targets = OrderedSet()
for target in self.targets:
if is_jvm(target):
jvm_targets.add(target)
elif is_python(target):
python_targets.add(target)
else:
self.error("Cannot build target %s" % target)
if jvm_targets:
status = self._jvm_build(jvm_targets)
if status != 0:
return status
if python_targets:
status = self._python_build(python_targets)
return status
class InternalTarget(Target):
"""A baseclass for targets that support an optional dependency set."""
@classmethod
def check_cycles(cls, internal_target):
"""Validates the given InternalTarget has no circular dependencies. Raises CycleException if
it does."""
dep_stack = OrderedSet()
def descend(internal_dep):
if internal_dep in dep_stack:
raise CycleException(dep_stack, internal_dep)
if hasattr(internal_dep, 'internal_dependencies'):
dep_stack.add(internal_dep)
for dep in internal_dep.internal_dependencies:
descend(dep)
dep_stack.remove(internal_dep)
descend(internal_target)
@classmethod
def sort_targets(cls, internal_targets):
"""Returns a list of targets that internal_targets depend on sorted from most dependent to
least."""
roots = OrderedSet()
inverted_deps = collections.defaultdict(OrderedSet) # target -> dependent targets
visited = set()
def invert(target):
if target not in visited:
visited.add(target)
if target.internal_dependencies:
for internal_dependency in target.internal_dependencies:
if isinstance(internal_dependency, InternalTarget):
inverted_deps[internal_dependency].add(target)
invert(internal_dependency)
else:
roots.add(target)
for internal_target in internal_targets:
invert(internal_target)
sorted = []
visited.clear()
def topological_sort(target):
if target not in visited:
visited.add(target)
if target in inverted_deps:
for dep in inverted_deps[target]:
topological_sort(dep)
sorted.append(target)
for root in roots:
topological_sort(root)
return sorted
@classmethod
def coalesce_targets(cls, internal_targets):
"""Returns a list of targets internal_targets depend on sorted from most dependent to least and
grouped where possible by target type."""
sorted_targets = InternalTarget.sort_targets(internal_targets)
# can do no better for any of these:
# []
# [a]
# [a,b]
if len(sorted_targets) <= 2:
return sorted_targets
# For these, we'd like to coalesce if possible, like:
# [a,b,a,c,a,c] -> [a,a,a,b,c,c]
# adopt a quadratic worst case solution, when we find a type change edge, scan forward for
# the opposite edge and then try to swap dependency pairs to move the type back left to its
# grouping. If the leftwards migration fails due to a dependency constraint, we just stop
# and move on leaving "type islands".
current_type = None
# main scan left to right no backtracking
for i in range(len(sorted_targets) - 1):
current_target = sorted_targets[i]
if current_type != type(current_target):
scanned_back = False
# scan ahead for next type match
for j in range(i + 1, len(sorted_targets)):
look_ahead_target = sorted_targets[j]
if current_type == type(look_ahead_target):
scanned_back = True
# swap this guy as far back as we can
for k in range(j, i, -1):
previous_target = sorted_targets[k - 1]
mismatching_types = current_type != type(previous_target)
not_a_dependency = look_ahead_target not in previous_target.internal_dependencies
if mismatching_types and not_a_dependency:
sorted_targets[k] = sorted_targets[k - 1]
sorted_targets[k - 1] = look_ahead_target
else:
break # out of k
break # out of j
if not scanned_back: # done with coalescing the current type, move on to next
current_type = type(current_target)
return sorted_targets
def sort(self):
"""Returns a list of targets this target depends on sorted from most dependent to least."""
return InternalTarget.sort_targets([ self ])
def coalesce(self):
"""Returns a list of targets this target depends on sorted from most dependent to least and
grouped where possible by target type."""
return InternalTarget.coalesce_targets([ self ])
def __init__(self, name, dependencies, is_meta):
Target.__init__(self, name, is_meta)
self.resolved_dependencies = OrderedSet()
self.internal_dependencies = OrderedSet()
self.jar_dependencies = OrderedSet()
self.update_dependencies(dependencies)
def update_dependencies(self, dependencies):
if dependencies:
for dependency in dependencies:
for resolved_dependency in dependency.resolve():
self.resolved_dependencies.add(resolved_dependency)
if isinstance(resolved_dependency, InternalTarget):
self.internal_dependencies.add(resolved_dependency)
self.jar_dependencies.update(resolved_dependency._as_jar_dependencies())
def _walk(self, walked, work, predicate = None):
Target._walk(self, walked, work, predicate)
for dep in self.resolved_dependencies:
if isinstance(dep, Target) and not dep in walked:
walked.add(dep)
if not predicate or predicate(dep):
additional_targets = work(dep)
dep._walk(walked, work, predicate)
if additional_targets:
for additional_target in additional_targets:
additional_target._walk(walked, work, predicate)
class InternalTarget(Target):
"""A baseclass for targets that support an optional dependency set."""
def sort(self):
"""Returns a list of targets this target depends on sorted from most dependent to least."""
roots = dict() # address -> root target
inverted_deps = collections.defaultdict(OrderedSet) # address -> dependent targets
visited = set() # addresses
def invert(target):
if target.address not in visited:
visited.add(target.address)
if target.internal_dependencies:
for internal_dependency in target.internal_dependencies:
if isinstance(internal_dependency, InternalTarget):
inverted_deps[internal_dependency.address].add(target)
invert(internal_dependency)
else:
roots[target.address] = target
invert(self)
sorted = []
visited.clear()
def topological_sort(target):
if target.address not in visited:
visited.add(target.address)
if target.address in inverted_deps:
for dep in inverted_deps[target.address]:
topological_sort(dep)
sorted.append(target)
for root in roots.values():
topological_sort(root)
return sorted
def coalesce(self):
"""Returns a list of targets this target depends on sorted from most dependent to least and
grouped where possible by target type."""
sorted_targets = self.sort()
# can do no better for any of these:
# []
# [a]
# [a,b]
if len(sorted_targets) <= 2:
return sorted_targets
# For these, we'd like to coalesce if possible, like:
# [a,b,a,c,a,c] -> [a,a,a,b,c,c]
# adopt a quadratic worst case solution, when we find a type change edge, scan forward for
# the opposite edge and then try to swap dependency pairs to move the type back left to its
# grouping. If the leftwards migration fails due to a dependency constraint, we just stop
# and move on leaving "type islands".
current_type = type(self)
# main scan left to right no backtracking
for i in range(len(sorted_targets) - 1):
current_target = sorted_targets[i]
if current_type != type(current_target):
scanned_back = False
# scan ahead for next type match
for j in range(i + 1, len(sorted_targets)):
look_ahead_target = sorted_targets[j]
if current_type == type(look_ahead_target):
scanned_back = True
# swap this guy as far back as we can
for k in range(j, i, -1):
previous_target = sorted_targets[k - 1]
mismatching_types = current_type != type(previous_target)
not_a_dependency = look_ahead_target not in previous_target.internal_dependencies
if mismatching_types and not_a_dependency:
sorted_targets[k] = sorted_targets[k - 1]
sorted_targets[k - 1] = look_ahead_target
else:
break # out of k
break # out of j
if not scanned_back: # done with coalescing the current type, move on to next
current_type = type(current_target)
return sorted_targets
def __init__(self, name, dependencies, is_meta):
Target.__init__(self, name, is_meta)
self.resolved_dependencies = OrderedSet()
self.internal_dependencies = OrderedSet()
self.jar_dependencies = OrderedSet()
if dependencies:
for dependency in dependencies:
resolved_dependency = dependency.resolve()
self.resolved_dependencies.add(resolved_dependency)
if isinstance(resolved_dependency, InternalTarget):
self.internal_dependencies.add(resolved_dependency)
self.jar_dependencies.update(resolved_dependency._as_jar_dependencies())
class Build(Command):
"""Builds a specified target."""
def setup_parser(self, parser):
parser.set_usage("\n"
" %prog build (options) [spec] (build args)\n"
" %prog build (options) [spec]... -- (build args)")
parser.disable_interspersed_args()
parser.add_option(
"--fast",
action="store_true",
dest="is_meta",
default=False,
help=
"Specifies the build should be flattened before executing, this can "
"help speed up many builds. Equivalent to the ! suffix BUILD target "
"modifier")
parser.add_option(
"--ide",
action="store_true",
dest="is_ide",
default=False,
help=
"Specifies the build should just do enough to get an IDE usable.")
parser.add_option(
"-t",
"--ide-transitivity",
dest="ide_transitivity",
type="choice",
choices=_TRANSITIVITY_CHOICES,
default=TRANSITIVITY_TESTS,
help=
"[%%default] Specifies IDE dependencies should be transitive for one "
"of: %s" % _TRANSITIVITY_CHOICES)
parser.add_option("-q",
"--quiet",
action="store_true",
dest="quiet",
default=False,
help="Don't output result of empty targets")
parser.epilog = """Builds the specified target(s). Currently any additional arguments are
passed straight through to the ant build system."""
def __init__(self, root_dir, parser, argv):
Command.__init__(self, root_dir, parser, argv)
if not self.args:
self.error("A spec argument is required")
try:
specs_end = self.args.index('--')
if len(self.args) > specs_end:
self.build_args = self.args.__getslice__(
specs_end + 1,
len(self.args) + 1)
else:
self.build_args = []
except ValueError:
specs_end = 1
self.build_args = self.args[1:] if len(self.args) > 1 else []
self.targets = OrderedSet()
for spec in self.args.__getslice__(0, specs_end):
try:
address = Address.parse(root_dir, spec)
except:
self.error("Problem parsing spec %s: %s" %
(spec, traceback.format_exc()))
try:
target = Target.get(address)
except:
self.error("Problem parsing BUILD target %s: %s" %
(address, traceback.format_exc()))
try:
InternalTarget.check_cycles(target)
except CycleException as e:
self.error("Target contains an internal dependency cycle: %s" %
e)
if not target:
self.error("Target %s does not exist" % address)
if not target.address.is_meta:
target.address.is_meta = self.options.is_meta or address.is_meta
self.targets.add(target)
self.is_ide = self.options.is_ide
self.ide_transitivity = self.options.ide_transitivity
def execute(self):
print "Build operating on targets: %s" % self.targets
jvm_targets = OrderedSet()
python_targets = OrderedSet()
for target in self.targets:
if is_jvm(target):
jvm_targets.add(target)
elif is_python(target):
python_targets.add(target)
else:
self.error("Cannot build target %s" % target)
if jvm_targets:
status = self._jvm_build(jvm_targets)
if status != 0:
return status
if python_targets:
status = self._python_build(python_targets)
return status
def _jvm_build(self, targets):
try:
# TODO(John Sirois): think about moving away from the ant backend
executor = AntBuilder(self.error, self.root_dir, self.is_ide,
self.ide_transitivity)
if self.options.quiet:
self.build_args.insert(0, "-logger")
self.build_args.insert(1,
"org.apache.tools.ant.NoBannerLogger")
self.build_args.insert(2, "-q")
return executor.build(targets, self.build_args)
except:
self.error("Problem executing AntBuilder for targets %s: %s" %
(targets, traceback.format_exc()))
def _python_build(self, targets):
try:
executor = PythonBuilder(self.error, self.root_dir)
return executor.build(targets, self.build_args)
except:
self.error("Problem executing PythonBuilder for targets %s: %s" %
(targets, traceback.format_exc()))
def configure(self):
"""Configures this project's source sets returning the full set of targets the project is
comprised of. The full set can be larger than the initial set of targets when any of the
initial targets only has partial ownership of its parent directory source set."""
analyzed = OrderedSet()
targeted = set()
def accept_target(target):
return has_sources(target) and not target.is_codegen
def configure_source_sets(relative_base, sources, is_test):
absolute_base = os.path.join(self.root_dir, relative_base)
paths = set([ os.path.dirname(source) for source in sources])
for path in paths:
absolute_path = os.path.join(absolute_base, path)
if absolute_path not in targeted:
targeted.add(absolute_path)
self.sources.append(SourceSet(self.root_dir, relative_base, path, is_test))
def configure_target(target):
if target not in analyzed:
analyzed.add(target)
self.has_scala = self.has_scala or is_scala(target)
if isinstance(target, JavaLibrary) or isinstance(target, ScalaLibrary):
# TODO(John Sirois): this does not handle test resources, make test resources 1st class
# in ant build and punch this through to pants model
resources = set()
if target.resources:
resources.update(target.resources)
if target.binary_resources:
resources.update(target.binary_resources)
if resources:
self.resource_extensions.update(Project.extract_resource_extensions(resources))
configure_source_sets(RESOURCES_BASE_DIR, resources, is_test = False)
if target.sources:
test = is_test(target)
self.has_tests = self.has_tests or test
configure_source_sets(target.target_base, target.sources, is_test = test)
siblings = Target.get_all_addresses(target.address.buildfile)
return filter(accept_target, [ Target.get(a) for a in siblings if a != target.address ])
for target in self.targets:
target.walk(configure_target, predicate = accept_target)
for source_set in self.sources:
paths = set()
source_base = os.path.join(self.root_dir, source_set.source_base)
for root, dirs, _ in os.walk(os.path.join(source_base, source_set.path)):
if dirs:
paths.update([ os.path.join(root, dir) for dir in dirs ])
unused_children = paths - targeted
if unused_children and paths != unused_children:
source_set.excludes.extend(os.path.relpath(child, source_base) for child in unused_children)
targets = OrderedSet()
for target in self.targets:
target.walk(lambda target: targets.add(target), has_sources)
targets.update(analyzed - targets)
return targets
def configure(self):
"""Configures this project's source sets returning the full set of targets the project is
comprised of. The full set can be larger than the initial set of targets when any of the
initial targets only has partial ownership of its parent directory source set."""
analyzed = OrderedSet()
targeted = set()
def accept_target(target):
return has_sources(target) and not target.is_codegen
def configure_source_sets(relative_base, sources, is_test):
absolute_base = os.path.join(self.root_dir, relative_base)
paths = set([os.path.dirname(source) for source in sources])
for path in paths:
absolute_path = os.path.join(absolute_base, path)
if absolute_path not in targeted:
targeted.add(absolute_path)
self.sources.append(
SourceSet(self.root_dir, relative_base, path, is_test))
def configure_target(target):
if target not in analyzed:
analyzed.add(target)
self.has_scala = self.has_scala or is_scala(target)
if isinstance(target, JavaLibrary) or isinstance(
target, ScalaLibrary):
# TODO(John Sirois): this does not handle test resources, make test resources 1st class
# in ant build and punch this through to pants model
resources = set()
if target.resources:
resources.update(target.resources)
if target.binary_resources:
resources.update(target.binary_resources)
if resources:
self.resource_extensions.update(
Project.extract_resource_extensions(resources))
configure_source_sets(RESOURCES_BASE_DIR,
resources,
is_test=False)
if target.sources:
test = is_test(target)
self.has_tests = self.has_tests or test
configure_source_sets(target.target_base,
target.sources,
is_test=test)
siblings = Target.get_all_addresses(target.address.buildfile)
return filter(
accept_target,
[Target.get(a) for a in siblings if a != target.address])
for target in self.targets:
target.walk(configure_target, predicate=accept_target)
for source_set in self.sources:
paths = set()
source_base = os.path.join(self.root_dir, source_set.source_base)
for root, dirs, _ in os.walk(
os.path.join(source_base, source_set.path)):
if dirs:
paths.update([os.path.join(root, dir) for dir in dirs])
unused_children = paths - targeted
if unused_children and paths != unused_children:
source_set.excludes.extend(
os.path.relpath(child, source_base)
for child in unused_children)
targets = OrderedSet()
for target in self.targets:
target.walk(lambda target: targets.add(target), has_sources)
targets.update(analyzed - targets)
return targets
class Build(Command):
"""Builds a specified target."""
def setup_parser(self, parser):
parser.set_usage("\n"
" %prog build (options) [spec] (build args)\n"
" %prog build (options) [spec]... -- (build args)")
parser.disable_interspersed_args()
parser.add_option("--fast", action="store_true", dest = "is_meta", default = False,
help = "Specifies the build should be flattened before executing, this can "
"help speed up many builds. Equivalent to the ! suffix BUILD target "
"modifier")
parser.add_option("--ide", action="store_true", dest = "is_ide", default = False,
help = "Specifies the build should just do enough to get an IDE usable.")
parser.add_option("-t", "--ide-transitivity", dest = "ide_transitivity", type = "choice",
choices = _TRANSITIVITY_CHOICES, default = TRANSITIVITY_TESTS,
help = "[%%default] Specifies IDE dependencies should be transitive for one "
"of: %s" % _TRANSITIVITY_CHOICES)
parser.add_option("-q", "--quiet", action="store_true", dest = "quiet", default = False,
help = "Don't output result of empty targets")
parser.epilog = """Builds the specified target(s). Currently any additional arguments are
passed straight through to the ant build system."""
def __init__(self, root_dir, parser, argv):
Command.__init__(self, root_dir, parser, argv)
if not self.args:
self.error("A spec argument is required")
try:
specs_end = self.args.index('--')
if len(self.args) > specs_end:
self.build_args = self.args.__getslice__(specs_end + 1, len(self.args) + 1)
else:
self.build_args = []
except ValueError:
specs_end = 1
self.build_args = self.args[1:] if len(self.args) > 1 else []
self.targets = OrderedSet()
for spec in self.args.__getslice__(0, specs_end):
try:
address = Address.parse(root_dir, spec)
except:
self.error("Problem parsing spec %s: %s" % (spec, traceback.format_exc()))
try:
target = Target.get(address)
except:
self.error("Problem parsing BUILD target %s: %s" % (address, traceback.format_exc()))
try:
InternalTarget.check_cycles(target)
except CycleException as e:
self.error("Target contains an internal dependency cycle: %s" % e)
if not target:
self.error("Target %s does not exist" % address)
if not target.address.is_meta:
target.address.is_meta = self.options.is_meta or address.is_meta
self.targets.add(target)
self.is_ide = self.options.is_ide
self.ide_transitivity = self.options.ide_transitivity
def execute(self):
print "Build operating on targets: %s" % self.targets
jvm_targets = OrderedSet()
python_targets = OrderedSet()
for target in self.targets:
if is_jvm(target):
jvm_targets.add(target)
elif is_python(target):
python_targets.add(target)
else:
self.error("Cannot build target %s" % target)
if jvm_targets:
status = self._jvm_build(jvm_targets)
if status != 0:
return status
if python_targets:
status = self._python_build(python_targets)
return status
def _jvm_build(self, targets):
try:
# TODO(John Sirois): think about moving away from the ant backend
executor = AntBuilder(self.error, self.root_dir, self.is_ide, self.ide_transitivity)
if self.options.quiet:
self.build_args.insert(0, "-logger")
self.build_args.insert(1, "org.apache.tools.ant.NoBannerLogger")
self.build_args.insert(2, "-q")
return executor.build(targets, self.build_args)
except:
self.error("Problem executing AntBuilder for targets %s: %s" % (targets,
traceback.format_exc()))
def _python_build(self, targets):
try:
executor = PythonBuilder(self.error, self.root_dir)
return executor.build(targets, self.build_args)
except:
self.error("Problem executing PythonBuilder for targets %s: %s" % (targets,
traceback.format_exc()))