def __init__(self, name, sources=None, exclusives=None):
Target.__init__(self, name, exclusives=exclusives)
self.add_labels("sources")
self.target_base = SourceRoot.find(self)
self.sources = None
self._sources = sources or []
def __init__(self, requirement, dynamic=False, repository=None, name=None, version_filter=None): self._requirement = Requirement.parse(requirement) self._name = name or self._requirement.project_name self._dynamic = dynamic self._repository = repository self._version_filter = version_filter or (lambda: True) Target.__init__(self, self._name, False)
def execute(self):
for buildfile in BuildFile.scan_buildfiles(self.root_dir):
for address in Target.get_all_addresses(buildfile):
target = Target.get(address)
if hasattr(target, 'sources') and target.sources is not None:
for sourcefile in target.sources:
print sourcefile, address
def configure_target(target):
if target not in analyzed:
analyzed.add(target)
self.has_scala = self.has_scala or is_scala(target)
if isinstance(target, JavaLibrary) or isinstance(target, ScalaLibrary):
# TODO(John Sirois): this does not handle test resources, make test resources 1st class
# in ant build and punch this through to pants model
resources = set()
if target.resources:
resources.update(target.resources)
if target.binary_resources:
resources.update(target.binary_resources)
if resources:
self.resource_extensions.update(Project.extract_resource_extensions(resources))
configure_source_sets(ExportableJvmLibrary.RESOURCES_BASE_DIR, resources, is_test = False)
if target.sources:
test = is_test(target)
self.has_tests = self.has_tests or test
configure_source_sets(target.target_base, target.sources, is_test = test)
siblings = Target.get_all_addresses(target.address.buildfile)
return filter(accept_target, [ Target.get(a) for a in siblings if a != target.address ])
def test_sibling_references(self):
with temporary_dir() as root_dir:
buildfile = create_buildfile(root_dir, 'a', name='BUILD',
content=dedent("""
dependencies(name='util',
dependencies=[
jar(org='com.twitter', name='util', rev='0.0.1')
]
)
""").strip()
)
sibling = create_buildfile(root_dir, 'a', name='BUILD.sibling',
content=dedent("""
dependencies(name='util-ex',
dependencies=[
pants(':util'),
jar(org='com.twitter', name='util-ex', rev='0.0.1')
]
)
""").strip()
)
ParseContext(buildfile).parse()
utilex = Target.get(Address.parse(root_dir, 'a:util-ex', is_relative=False))
utilex_deps = set(utilex.resolve())
util = Target.get(Address.parse(root_dir, 'a:util', is_relative=False))
util_deps = set(util.resolve())
self.assertEquals(util_deps, util_deps.intersection(utilex_deps))
def __init__(self, name, sources=None):
Target.__init__(self, name)
self.add_labels('sources')
self.target_base = SourceRoot.find(self)
self.sources = None
self._sources = sources or []
def __init__(self, name, dependencies, is_meta):
Target.__init__(self, name, is_meta)
self._injected_deps = []
self.processed_dependencies = resolve(dependencies)
self.add_label('internal')
self.dependency_addresses = OrderedSet()
self.dependencies = OrderedSet()
self.internal_dependencies = OrderedSet()
self.jar_dependencies = OrderedSet()
# TODO(John Sirois): if meta targets were truly built outside parse contexts - we could instead
# just use the more general check: if parsing: delay(doit) else: doit()
# Fix how target _ids are built / addresses to not require a BUILD file - ie: support anonymous,
# non-addressable targets - which is what meta-targets really are once created.
if is_meta:
# Meta targets are built outside any parse context - so update dependencies immediately
self.update_dependencies(self.processed_dependencies)
else:
# Defer dependency resolution after parsing the current BUILD file to allow for forward
# references
self._post_construct(self.update_dependencies, self.processed_dependencies)
self._post_construct(self.inject_dependencies)
def add_targets(self, error, dir, buildfile):
try:
self.targets.extend(Target.get(addr) for addr in Target.get_all_addresses(buildfile))
except (TypeError, ImportError):
error(dir, include_traceback=True)
except (IOError, SyntaxError):
error(dir)
def __init__(self, name, dependencies, is_meta):
Target.__init__(self, name, is_meta)
self.resolved_dependencies = OrderedSet()
self.internal_dependencies = OrderedSet()
self.jar_dependencies = OrderedSet()
self.update_dependencies(dependencies)
def __init__(self, name, url_builder, exclusives=None):
"""
:param string name: The name of this target, which combined with this
build file defines the target :class:`twitter.pants.base.address.Address`.
:param url_builder: Function that accepts a page target and an optional wiki config dict.
:returns: A tuple of (alias, fully qualified url).
"""
Target.__init__(self, name, exclusives=exclusives)
self.url_builder = url_builder
def __init__(self, name, username=None, password=None):
"""
:name The name of these credentials
:username Either a constant username value or else a callable that can fetch one
:password Either a constant password value or else a callable that can fetch one
"""
Target.__init__(self, name, False)
self._username = username if callable(username) else lambda: username
self._password = password if callable(password) else lambda: password
def _find_targets(self):
if len(self.context.target_roots) > 0:
for target in self.context.target_roots:
yield target
else:
for buildfile in BuildFile.scan_buildfiles(get_buildroot()):
target_addresses = Target.get_all_addresses(buildfile)
for target_address in target_addresses:
yield Target.get(target_address)
def __init__(self, name, url, push_db, exclusives=None):
"""name: an identifier for the repo
url: the url used to access the repo and retrieve artifacts or artifact metadata
push_db: the data file associated with this repo that records artifact push history"""
Target.__init__(self, name, exclusives=exclusives)
self.name = name
self.url = url
self.push_db = push_db
def __init__(self, name, dependencies):
"""name: The name of this module target, addressable via pants via the portion of the spec
following the colon
dependencies: one or more JarDependencies this JarLibrary bundles or Pants pointing to other
JarLibraries or JavaTargets"""
assert len(dependencies) > 0, "At least one dependency must be specified"
Target.__init__(self, name, False)
self.dependencies = dependencies
def execute(self, expanded_target_addresses):
buildroot = get_buildroot()
if len(self.context.target_roots) > 0:
for target in self.context.target_roots:
self._execute_target(target, buildroot)
else:
for buildfile in BuildFile.scan_buildfiles(buildroot):
target_addresses = Target.get_all_addresses(buildfile)
for target_address in target_addresses:
target = Target.get(target_address)
self._execute_target(target, buildroot)
def _walk(self, walked, work, predicate = None):
Target._walk(self, walked, work, predicate)
for dep in self.dependencies:
if isinstance(dep, Target) and not dep in walked:
walked.add(dep)
if not predicate or predicate(dep):
additional_targets = work(dep)
dep._walk(walked, work, predicate)
if additional_targets:
for additional_target in additional_targets:
additional_target._walk(walked, work, predicate)
def __init__(self, requirement, name=None, repository=None, version_filter=None, use_2to3=False,
compatibility=None, exclusives=None):
# TODO(wickman) Allow PythonRequirements to be specified using pip-style vcs or url identifiers,
# e.g. git+https or just http://...
self._requirement = Requirement.parse(requirement)
self._repository = repository
self._name = name or self._requirement.project_name
self._use_2to3 = use_2to3
self._version_filter = version_filter or (lambda py, pl: True)
# TODO(wickman) Unify this with PythonTarget .compatibility
self.compatibility = compatibility or ['']
Target.__init__(self, self._name, exclusives=exclusives)
def _parse_addresses(self, spec):
if spec.endswith('::'):
dir = self._get_dir(spec[:-len('::')])
for buildfile in BuildFile.scan_buildfiles(self._root_dir, os.path.join(self._root_dir, dir)):
for address in Target.get_all_addresses(buildfile):
yield address
elif spec.endswith(':'):
dir = self._get_dir(spec[:-len(':')])
for address in Target.get_all_addresses(BuildFile(self._root_dir, dir)):
yield address
else:
yield Address.parse(self._root_dir, spec)
def __init__(self, name, dependencies):
"""name: The name of this module target, addressable via pants via the portion of the spec
following the colon
dependencies: one or more JarDependencies this JarLibrary bundles or Pants pointing to other
JarLibraries or JavaTargets"""
assert len(dependencies) > 0, "At least one dependency must be specified"
Target.__init__(self, name, False)
self.add_label('jars')
self.dependencies = resolve(dependencies)
self.dependency_addresses = set()
for dependency in self.dependencies:
if hasattr(dependency, 'address'):
self.dependency_addresses.add(dependency.address)
def __init__(self, spec):
# it's critical the spec is parsed 1st, the results are needed elsewhere in constructor flow
parse_context = ParseContext.locate()
def parse_address():
if spec.startswith(':'):
# the :[target] could be in a sibling BUILD - so parse using the canonical address
pathish = "%s:%s" % (parse_context.buildfile.canonical_relpath, spec[1:])
return Address.parse(parse_context.buildfile.root_dir, pathish, False)
else:
return Address.parse(parse_context.buildfile.root_dir, spec, False)
self.address = parse_address()
Target.__init__(self, self.address.target_name, False)
def __init__(self, context):
ConsoleTask.__init__(self, context)
self._print_uptodate = context.options.check_deps_print_uptodate
self.repos = context.config.getdict('jar-publish', 'repos')
self._artifacts_to_targets = {}
all_addresses = (address for buildfile in BuildFile.scan_buildfiles(get_buildroot())
for address in Target.get_all_addresses(buildfile))
for address in all_addresses:
target = Target.get(address)
if target.is_exported:
provided_jar, _, _ = target.get_artifact_info()
artifact = (provided_jar.org, provided_jar.name)
if not artifact in self._artifacts_to_targets:
self._artifacts_to_targets[artifact] = target
def _find_path(cls, from_target, to_target, log):
from_target, to_target = cls._coerce_to_targets(from_target, to_target)
log.debug('Looking for path from %s to %s' % (from_target.address.reference(), to_target.address.reference()))
queue = [([from_target], 0)]
while True:
if not queue:
print('no path found from %s to %s!' % (from_target.address.reference(), to_target.address.reference()))
break
path, indent = queue.pop(0)
next_target = path[-1]
if next_target in cls.examined_targets:
continue
cls.examined_targets.add(next_target)
log.debug('%sexamining %s' % (' ' * indent, next_target))
if next_target == to_target:
print('')
for target in path:
print('%s' % target.address.reference())
break
if hasattr(next_target, 'dependency_addresses'):
for address in next_target.dependency_addresses:
dep = Target.get(address)
queue.append((path + [dep], indent + 1))
def execute(self):
if self.options.only_provides:
def extract_artifact_id(target):
provided_jar = target._as_jar_dependency()
return "%s%s%s" % (provided_jar.org, self.options.separator, provided_jar.name)
extractors = dict(
address = lambda target: str(target.address),
artifact_id = extract_artifact_id,
repo_name = lambda target: target.provides.repo.name,
repo_url = lambda target: target.provides.repo.url,
repo_db = lambda target: target.provides.repo.push_db,
)
column_extractors = [ extractors[col] for col in (self.options.provides_columns.split(',')) ]
print_fn = lambda address: self._print_provides(column_extractors, address)
elif self.options.documented:
def print_documented(address):
target = Target.get(address)
if target.description:
return '%s\n %s' % (address, '\n '.join(target.description.strip().split('\n')))
print_fn = print_documented
else:
print_fn = lambda address: str(address)
for buildfile in self.buildfiles:
for address in Target.get_all_addresses(buildfile):
line = print_fn(address)
if line:
print(line)
def _find_paths(cls, from_target, to_target, log, find_all):
from_target, to_target = cls._coerce_to_targets(from_target, to_target)
log.debug('Looking for path from %s to %s' % (from_target.address.reference(), to_target.address.reference()))
paths_found = False
queue = [([from_target], 0)]
while True:
if not queue:
if not paths_found:
print 'no path found from %s to %s!' % (from_target.address.reference(), to_target.address.reference())
break
path, indent = queue.pop(0)
next_target = path[-1]
log.debug('%sexamining %s' % (' ' * indent, next_target))
if next_target == to_target:
if paths_found:
print ''
else:
paths_found = True
for target in path:
print '%s' % target.address.reference()
if find_all:
continue
else:
break
if hasattr(next_target, 'dependency_addresses'):
for address in next_target.dependency_addresses:
dep = Target.get(address)
queue.append((path + [dep], indent + 1))
def __init__(self, root_dir, parser, argv):
Command.__init__(self, root_dir, parser, argv)
if not self.args:
self.error("A spec argument is required")
targets = []
for k in range(len(self.args)):
arg = self.args[0]
if arg == '--':
self.args.pop(0)
break
try:
address = Address.parse(root_dir, arg)
target = Target.get(address)
except Exception as e:
break
if not target:
break
targets.append(target)
self.args.pop(0)
# stop at PythonBinary target
if isinstance(target, PythonBinary):
break
self.target = targets.pop(0) if targets else None
self.extra_targets = targets
if self.target is None:
self.error('No valid target specified!')
def resolve(self):
# De-reference this pants pointer to an actual parsed target.
resolved = Target.get(self.address)
if not resolved:
raise KeyError("Failed to find target for: %s" % self.address)
for dep in resolved.resolve():
yield dep
def __init__(self, spec):
# it's critical the spec is parsed 1st, the results are needed elsewhere in constructor flow
parse_context = ParseContext.locate()
def parse_address():
if spec.startswith(':'):
# the :[target] could be in a sibling BUILD - so parse using the canonical address
pathish = "%s:%s" % (parse_context.buildfile.canonical_relpath, spec[1:])
return Address.parse(parse_context.buildfile.root_dir, pathish, False)
else:
return Address.parse(parse_context.buildfile.root_dir, spec, False)
self.address = parse_address()
# We must disable the re-init check, because our funky __getattr__ breaks it.
# We're not involved in any multiple inheritance, so it's OK to disable it here.
Target.__init__(self, self.address.target_name, False, reinit_check=False)
def __init__(self, root_dir, parser, argv):
Command.__init__(self, root_dir, parser, argv)
if not self.args:
self.error("A spec argument is required")
try:
specs_end = self.args.index('--')
if len(self.args) > specs_end:
self.build_args = self.args[specs_end+1:len(self.args)+1]
else:
self.build_args = []
except ValueError:
specs_end = 1
self.build_args = self.args[1:] if len(self.args) > 1 else []
self.targets = OrderedSet()
for spec in self.args[0:specs_end]:
try:
address = Address.parse(root_dir, spec)
except:
self.error("Problem parsing spec %s: %s" % (spec, traceback.format_exc()))
try:
target = Target.get(address)
except:
self.error("Problem parsing BUILD target %s: %s" % (address, traceback.format_exc()))
if not target:
self.error("Target %s does not exist" % address)
self.targets.update(tgt for tgt in target.resolve() if is_concrete(tgt))
def _parse_targets(self, targets, root_dir):
for spec in self.args:
try:
address = Address.parse(root_dir, spec)
except:
self.error("Problem parsing spec %s: %s" % (spec, traceback.format_exc()))
try:
target = Target.get(address)
except:
self.error("Problem parsing target %s: %s" % (address, traceback.format_exc()))
if address.is_meta:
print("target is meta")
target = target.do_in_context(lambda: bang.extract_target([target], None))
if not IvyResolve._is_resolvable(target):
self.error("Target: %s is not resolvable" % address)
targets.add(target)
if not self.intransitive:
def add_targets(ttarget):
if hasattr(ttarget, 'internal_dependencies'):
for dep in ttarget.internal_dependencies:
if IvyResolve._is_resolvable(dep):
targets.add(dep)
else:
print("skipping %s as it's not ivy resolvable" % dep.name)
target.walk(add_targets)
return targets
def __getattr__(self, name):
try:
return Target.__getattribute__(self, name)
except AttributeError as e:
try:
return getattr(self.get(), name)
except (AttributeError, LookupError):
raise e
def java_sources(self):
if self._raw_java_sources is not None:
self._java_sources = list(
Target.resolve_all(maybe_list(self._raw_java_sources, Target),
JavaLibrary))
self._raw_java_sources = None
# TODO(John Sirois): reconsider doing this auto-linking.
# We have circular java/scala dep, add an inbound dependency edge from java to scala in this
# case to force scala compilation to precede java - since scalac supports generating java
# stubs for these cycles and javac does not this is both necessary and always correct.
for java_target in self._java_sources:
java_target.update_dependencies([self])
return self._java_sources
def __init__(self, spec, exclusives=None):
# it's critical the spec is parsed 1st, the results are needed elsewhere in constructor flow
parse_context = ParseContext.locate()
def parse_address():
if spec.startswith(':'):
# the :[target] could be in a sibling BUILD - so parse using the canonical address
pathish = "%s:%s" % (parse_context.buildfile.canonical_relpath,
spec[1:])
return Address.parse(parse_context.buildfile.root_dir, pathish,
False)
else:
return Address.parse(parse_context.buildfile.root_dir, spec,
False)
self.address = parse_address()
# We must disable the re-init check, because our funky __getattr__ breaks it.
# We're not involved in any multiple inheritance, so it's OK to disable it here.
Target.__init__(self,
self.address.target_name,
reinit_check=False,
exclusives=exclusives)
def __init__(self, run_tracker, root_dir, parser, argv):
Command.__init__(self, run_tracker, root_dir, parser, argv)
if not self.args:
self.error("A spec argument is required")
address = Address.parse(root_dir, self.args[0])
self.target = Target.get(address)
if self.target is None:
self.error('%s is not a valid target!' % self.args[0])
if not self.target.provides:
self.error('Target must provide an artifact.')
self.dependencies = self.minified_dependencies(self.target)
def __init__(self, root_dir, parser, argv):
Command.__init__(self, root_dir, parser, argv)
if not self.args:
self.error("A spec argument is required")
try:
address = Address.parse(root_dir, self.args[0])
target = Target.get(address)
except Exception as e:
self.error("Invalid target in %s (%s)" % (self.args[0], str(e)))
if not target:
self.error("Target %s does not exist" % address)
self.target = target
self.args.pop(0)
def __init__(self, root_dir, parser, argv):
Command.__init__(self, root_dir, parser, argv)
if not self.args:
self.error("A spec argument is required")
try:
specs_end = self.args.index('--')
if len(self.args) > specs_end:
self.build_args = self.args.__getslice__(
specs_end + 1,
len(self.args) + 1)
else:
self.build_args = []
except ValueError:
specs_end = 1
self.build_args = self.args[1:] if len(self.args) > 1 else []
self.targets = OrderedSet()
for spec in self.args.__getslice__(0, specs_end):
try:
address = Address.parse(root_dir, spec)
except:
self.error("Problem parsing spec %s: %s" %
(spec, traceback.format_exc()))
try:
target = Target.get(address)
except:
self.error("Problem parsing BUILD target %s: %s" %
(address, traceback.format_exc()))
try:
InternalTarget.check_cycles(target)
except CycleException as e:
self.error("Target contains an internal dependency cycle: %s" %
e)
if not target:
self.error("Target %s does not exist" % address)
if not target.address.is_meta:
target.address.is_meta = self.options.is_meta or address.is_meta
self.targets.add(target)
self.is_ide = self.options.is_ide
self.ide_transitivity = self.options.ide_transitivity
def _find_paths_rec(cls, from_target, to_target):
if from_target == to_target:
return [[from_target]]
if from_target not in cls.all_paths or to_target not in cls.all_paths[from_target]:
paths = []
if hasattr(from_target, 'dependency_addresses'):
for address in from_target.dependency_addresses:
dep = Target.get(address)
for path in cls._find_paths_rec(dep, to_target):
new_path = copy.copy(path)
new_path.insert(0, from_target)
paths.append(new_path)
cls.all_paths[from_target][to_target] = paths
return cls.all_paths[from_target][to_target]
def get_pytest_eggs(root):
specs = ["3rdparty/python:pytest"]
eggs = []
for spec in specs:
address = Address.parse(root, spec)
target = Target.get(address)
def add_eggs(target):
deps = []
for dep in target.dependencies:
if isinstance(dep, PythonEgg):
for egg in dep.eggs:
eggs.append(egg)
else:
deps.append(dep)
return deps
target.walk(lambda t: add_eggs(t))
return eggs
def _parse_targets(self, root_dir):
targets = OrderedSet()
for spec in self.args:
try:
address = Address.parse(root_dir, spec)
except:
self.error("Problem parsing spec %s: %s" % (spec, traceback.format_exc()))
try:
target = Target.get(address)
except:
self.error("Problem parsing target %s: %s" % (address, traceback.format_exc()))
if not Doc._is_documentable(target):
self.error("Target: %s is not documentable" % address)
targets.add(target)
return targets
def __init__(self, root_dir, parser, argv):
Command.__init__(self, root_dir, parser, argv)
if len(self.args) is not 1:
self.error("Exactly one BUILD address is required.")
spec = self.args[0]
try:
address = Address.parse(root_dir, spec)
except IOError:
self.error("Problem parsing spec %s: %s" %
(spec, traceback.format_exc()))
try:
self.target = Target.get(address)
except (ImportError, SyntaxError, TypeError):
self.error("Problem parsing BUILD target %s: %s" %
(address, traceback.format_exc()))
if not self.target:
self.error("Target %s does not exist" % address)
def _run_lint(self, target, args):
chroot = PythonChroot(target,
self.root_dir,
extra_targets=[
Target.get(
Address.parse(self.root_dir,
'3rdparty/python:pylint'))
])
builder = chroot.dump()
builder.info().entry_point = 'pylint.lint'
builder.freeze()
interpreter_args = [
'--rcfile=%s' %
os.path.join(self.root_dir, 'build-support', 'pylint', 'pylint.rc')
]
interpreter_args.extend(args or [])
sources = OrderedSet([])
target.walk(lambda trg: sources.update(trg.sources if hasattr(
trg, 'sources') and trg.sources is not None else []))
pex = PEX(builder.path())
pex.run(args=interpreter_args + list(sources), with_chroot=True)
def execute(self):
if self.options.only_provides:
def extract_artifact_id(target):
provided_jar = target._as_jar_dependency()
return "%s%s%s" % (provided_jar.org, self.options.separator, provided_jar.name)
extractors = dict(
address = lambda target: str(target.address),
artifact_id = extract_artifact_id,
repo_name = lambda target: target.provides.repo.name,
repo_url = lambda target: target.provides.repo.url,
repo_db = lambda target: target.provides.repo.push_db,
)
column_extractors = [ extractors[col] for col in (self.options.provides_columns.split(',')) ]
print_fn = lambda address: self._print_provides(column_extractors, address)
else:
print_fn = lambda address: str(address)
for buildfile in self.buildfiles:
for address in Target.get_all_addresses(buildfile):
line = print_fn(address)
if line:
print line
def setup_parser(self, parser, args):
self.config = Config.load()
Goal.add_global_options(parser)
# We support attempting zero or more goals. Multiple goals must be delimited from further
# options and non goal args with a '--'. The key permutations we need to support:
# ./pants goal => goals
# ./pants goal goals => goals
# ./pants goal compile src/java/... => compile
# ./pants goal compile -x src/java/... => compile
# ./pants goal compile src/java/... -x => compile
# ./pants goal compile run -- src/java/... => compile, run
# ./pants goal compile run -- src/java/... -x => compile, run
# ./pants goal compile run -- -x src/java/... => compile, run
if not args:
args.append('goals')
if len(args) == 1 and args[0] in set(['-h', '--help', 'help']):
def format_usage(usages):
left_colwidth = 0
for left, right in usages:
left_colwidth = max(left_colwidth, len(left))
lines = []
for left, right in usages:
lines.append(' %s%s%s' %
(left, ' ' *
(left_colwidth - len(left) + 1), right))
return '\n'.join(lines)
usages = [
("%prog goal goals ([spec]...)", Phase('goals').description),
("%prog goal help [goal] ([spec]...)",
Phase('help').description),
("%prog goal [goal] [spec]...",
"Attempt goal against one or more targets."),
("%prog goal [goal] ([goal]...) -- [spec]...",
"Attempts all the specified goals."),
]
parser.set_usage("\n%s" % format_usage(usages))
parser.epilog = (
"Either lists all installed goals, provides extra help for a goal or else "
"attempts to achieve the specified goal for the listed targets."
"""
Note that target specs accept two special forms:
[dir]: to include all targets in the specified directory
[dir]:: to include all targets found in all BUILD files recursively under
the directory""")
parser.print_help()
sys.exit(0)
else:
goals, specs = Goal.parse_args(args)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup',
labels=[WorkUnit.SETUP]):
# Bootstrap goals by loading any configured bootstrap BUILD files
with self.check_errors(
'The following bootstrap_buildfiles cannot be loaded:'
) as error:
with self.run_tracker.new_workunit(name='bootstrap',
labels=[WorkUnit.SETUP
]):
for path in self.config.getlist('goals',
'bootstrap_buildfiles',
default=[]):
try:
buildfile = BuildFile(
get_buildroot(),
os.path.relpath(path, get_buildroot()))
ParseContext(buildfile).parse()
except (TypeError, ImportError, TaskError,
GoalError):
error(path, include_traceback=True)
except (IOError, SyntaxError):
error(path)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
# Bootstrap user goals by loading any BUILD files implied by targets.
spec_parser = SpecParser(self.root_dir)
with self.check_errors(
'The following targets could not be loaded:') as error:
with self.run_tracker.new_workunit(name='parse',
labels=[WorkUnit.SETUP
]):
for spec in specs:
try:
for target, address in spec_parser.parse(spec):
if target:
self.targets.append(target)
# Force early BUILD file loading if this target is an alias that expands
# to others.
unused = list(target.resolve())
else:
siblings = Target.get_all_addresses(
address.buildfile)
prompt = 'did you mean' if len(
siblings
) == 1 else 'maybe you meant one of these'
error('%s => %s?:\n %s' %
(address, prompt, '\n '.join(
str(a) for a in siblings)))
except (TypeError, ImportError, TaskError,
GoalError):
error(spec, include_traceback=True)
except (IOError, SyntaxError,
TargetDefinitionException):
error(spec)
self.phases = [Phase(goal) for goal in goals]
rcfiles = self.config.getdefault('rcfiles', type=list, default=[])
if rcfiles:
rcfile = RcFile(rcfiles,
default_prepend=False,
process_default=True)
# Break down the goals specified on the command line to the full set that will be run so we
# can apply default flags to inner goal nodes. Also break down goals by Task subclass and
# register the task class hierarchy fully qualified names so we can apply defaults to
# baseclasses.
sections = OrderedSet()
for phase in Engine.execution_order(self.phases):
for goal in phase.goals():
sections.add(goal.name)
for clazz in goal.task_type.mro():
if clazz == Task:
break
sections.add('%s.%s' %
(clazz.__module__, clazz.__name__))
augmented_args = rcfile.apply_defaults(sections, args)
if augmented_args != args:
del args[:]
args.extend(augmented_args)
sys.stderr.write(
"(using pantsrc expansion: pants goal %s)\n" %
' '.join(augmented_args))
Phase.setup_parser(parser, args, self.phases)
def __init__(self, target_base, name, is_meta = False):
Target.__init__(self, name, is_meta)
# TODO(John Sirois): rationalize constructor parameter and find_target_base
self.target_base = target_base if target_base else self.find_target_base()
def get_targets():
for address in Command.scan_addresses(root_dir):
target = Target.get(address)
if IvyResolve._is_resolvable(target):
yield target
def execute(self, targets):
scala_targets = filter(lambda t: has_sources(t, '.scala'), targets)
if not scala_targets:
return
# Get the exclusives group for the targets to compile.
# Group guarantees that they'll be a single exclusives key for them.
egroups = self.context.products.get_data('exclusives_groups')
group_id = egroups.get_group_key_for_target(scala_targets[0])
# Add resource dirs to the classpath for us and for downstream tasks.
for conf in self._confs:
egroups.update_compatible_classpaths(group_id,
[(conf, self._resources_dir)])
# Get the classpath generated by upstream JVM tasks (including previous calls to execute()).
cp = egroups.get_classpath_for_group(group_id)
# Add (only to the local copy) classpath entries necessary for our compiler plugins.
for conf in self._confs:
for jar in self._zinc_utils.plugin_jars():
cp.insert(0, (conf, jar))
# Invalidation check. Everything inside the with block must succeed for the
# invalid targets to become valid.
with self.invalidated(scala_targets,
invalidate_dependents=True,
partition_size_hint=self._partition_size_hint
) as invalidation_check:
if invalidation_check.invalid_vts and not self.dry_run:
invalid_targets = [
vt.target for vt in invalidation_check.invalid_vts
]
# The analysis for invalid and deleted sources is no longer valid.
invalid_sources_by_target = self._compute_sources_by_target(
invalid_targets)
invalid_sources = list(
itertools.chain.from_iterable(
invalid_sources_by_target.values()))
deleted_sources = self._get_deleted_sources()
# Work in a tmpdir so we don't stomp the main analysis files on error.
# The tmpdir is cleaned up in a shutdown hook, because background work
# may need to access files we create here even after this method returns.
self._ensure_analysis_tmpdir()
tmpdir = os.path.join(self._analysis_tmpdir, str(uuid.uuid4()))
os.mkdir(tmpdir)
valid_analysis_tmp = os.path.join(tmpdir, 'valid_analysis')
newly_invalid_analysis_tmp = os.path.join(
tmpdir, 'newly_invalid_analysis')
invalid_analysis_tmp = os.path.join(tmpdir, 'invalid_analysis')
if ZincUtils.is_nonempty_analysis(self._analysis_file):
with self.context.new_workunit(name='prepare-analysis'):
if self._zinc_utils.run_zinc_split(
self._analysis_file,
((invalid_sources + deleted_sources,
newly_invalid_analysis_tmp),
([], valid_analysis_tmp))):
raise TaskError(
'Failed to split off invalid analysis.')
if ZincUtils.is_nonempty_analysis(
self._invalid_analysis_file):
if self._zinc_utils.run_zinc_merge([
self._invalid_analysis_file,
newly_invalid_analysis_tmp
], invalid_analysis_tmp):
raise TaskError(
'Failed to merge prior and current invalid analysis.'
)
else:
invalid_analysis_tmp = newly_invalid_analysis_tmp
# Now it's OK to overwrite the main analysis files with the new state.
ZincUtils._move_analysis(valid_analysis_tmp,
self._analysis_file)
ZincUtils._move_analysis(invalid_analysis_tmp,
self._invalid_analysis_file)
# Figure out the sources and analysis belonging to each partition.
partitions = [
] # Each element is a triple (vts, sources_by_target, analysis).
for vts in invalidation_check.invalid_vts_partitioned:
partition_tmpdir = os.path.join(
tmpdir, Target.maybe_readable_identify(vts.targets))
os.mkdir(partition_tmpdir)
sources = list(
itertools.chain.from_iterable([
invalid_sources_by_target.get(t, [])
for t in vts.targets
]))
analysis_file = os.path.join(partition_tmpdir, 'analysis')
partitions.append((vts, sources, analysis_file))
# Split per-partition files out of the global invalid analysis.
if ZincUtils.is_nonempty_analysis(
self._invalid_analysis_file) and partitions:
with self.context.new_workunit(name='partition-analysis'):
splits = [(x[1], x[2]) for x in partitions]
if self._zinc_utils.run_zinc_split(
self._invalid_analysis_file, splits):
raise TaskError(
'Failed to split invalid analysis into per-partition files.'
)
# Now compile partitions one by one.
for partition in partitions:
(vts, sources, analysis_file) = partition
self._process_target_partition(partition, cp)
# No exception was thrown, therefore the compile succeded and analysis_file is now valid.
if os.path.exists(
analysis_file
): # The compilation created an analysis.
# Kick off the background artifact cache write.
if self.get_artifact_cache(
) and self.context.options.write_to_artifact_cache:
self._write_to_artifact_cache(
analysis_file, vts, invalid_sources_by_target)
# Merge the newly-valid analysis into our global valid analysis.
if ZincUtils.is_nonempty_analysis(self._analysis_file):
with self.context.new_workunit(
name='update-upstream-analysis'):
new_valid_analysis = analysis_file + '.valid.new'
if self._zinc_utils.run_zinc_merge(
[self._analysis_file, analysis_file],
new_valid_analysis):
raise TaskError(
'Failed to merge new analysis back into valid analysis file.'
)
ZincUtils._move_analysis(new_valid_analysis,
self._analysis_file)
else: # We need to keep analysis_file around. Background tasks may need it.
ZincUtils._copy_analysis(analysis_file,
self._analysis_file)
if ZincUtils.is_nonempty_analysis(
self._invalid_analysis_file):
with self.context.new_workunit(
name='trim-downstream-analysis'):
# Trim out the newly-valid sources from our global invalid analysis.
new_invalid_analysis = analysis_file + '.invalid.new'
discarded_invalid_analysis = analysis_file + '.invalid.discard'
if self._zinc_utils.run_zinc_split(
self._invalid_analysis_file,
[(sources, discarded_invalid_analysis),
([], new_invalid_analysis)]):
raise TaskError(
'Failed to trim invalid analysis file.')
ZincUtils._move_analysis(
new_invalid_analysis,
self._invalid_analysis_file)
# Now that all the analysis accounting is complete, we can safely mark the
# targets as valid.
vts.update()
# Check for missing dependencies, if needed.
if invalidation_check.invalid_vts and os.path.exists(
self._analysis_file):
deps_cache = JvmDependencyCache(self.context,
scala_targets,
self._analysis_file,
self._classes_dir)
deps_cache.check_undeclared_dependencies()
# Provide the target->class and source->class mappings to downstream tasks if needed.
if self.context.products.isrequired('classes'):
sources_by_target = self._compute_sources_by_target(scala_targets)
classes_by_source = self._compute_classes_by_source()
self._add_all_products_to_genmap(sources_by_target,
classes_by_source)
# Update the classpath for downstream tasks.
for conf in self._confs:
egroups.update_compatible_classpaths(group_id,
[(conf, self._classes_dir)])
def _walk(self, walked, work, predicate=None): Target._walk(self, walked, work, predicate) self.binary._walk(walked, work, predicate)
def __init__(self, name, url_builder):
""":url_builder a function that accepts a page target and an optional wiki :config dict and
returns a tuple of (alias, fully qualified url)."""
Target.__init__(self, name)
self.url_builder = url_builder
def setUpClass(cls):
cls.build_root = mkdtemp(suffix='_BUILD_ROOT')
set_buildroot(cls.build_root)
cls._cwd = os.getcwd()
os.chdir(cls.build_root)
Target._clear_all_addresses()
def print_provides(column_extractors, address):
target = Target.get(address)
if target.is_exported:
return " ".join(
extractor(target) for extractor in column_extractors)
def __init__(self, name, url_builder, exclusives=None): """:url_builder a function that accepts a page target and an optional wiki :config dict and returns a tuple of (alias, fully qualified url).""" Target.__init__(self, name, is_meta=False, exclusives=exclusives) self.url_builder = url_builder
def _print_provides(self, column_extractors, address):
target = Target.get(address)
if is_exported(target):
return " ".join(extractor(target) for extractor in column_extractors)
def get_targets():
for address in Command.scan_addresses(root_dir):
target = Target.get(address)
if Doc._is_documentable(target):
yield target
def print_documented(address):
target = Target.get(address)
if target.description:
return '%s\n %s' % (address, '\n '.join(
target.description.strip().split('\n')))
def __init__(self, name, is_meta=False):
Target.__init__(self, name, is_meta)
self.add_label('sources')
self.target_base = SourceRoot.find(self)
def tearDown(self): Target._clear_all_addresses()
def __init__(
self,
name,
source=None,
dependencies=None,
entry_point=None,
inherit_path=False, # pex option
zip_safe=True, # pex option
always_write_cache=False, # pex option
repositories=None, # pex option
indices=None, # pex option
ignore_errors=False, # pex option
allow_pypi=False, # pex option
platforms=(),
compatibility=None,
exclusives=None):
"""
:param name: target name
:param source: the python source file that becomes this binary's __main__.
If None specified, drops into an interpreter by default.
:param dependencies: List of :class:`twitter.pants.base.target.Target` instances
this target depends on.
:type dependencies: list of targets
:param entry_point: the default entry point for this binary. if None, drops into the entry
point that is defined by source
:param inherit_path: inherit the sys.path of the environment that this binary runs in
:param zip_safe: whether or not this binary is safe to run in compacted (zip-file) form
:param always_write_cache: whether or not the .deps cache of this PEX file should always
be written to disk.
:param repositories: a list of repositories to query for dependencies.
:param indices: a list of indices to use for packages.
:param platforms: extra platforms to target when building this binary.
:param compatibility: either a string or list of strings that represents
interpreter compatibility for this target, using the Requirement-style format,
e.g. ``'CPython>=3', or just ['>=2.7','<3']`` for requirements agnostic to interpreter class.
:param dict exclusives: An optional dict of exclusives tags. See CheckExclusives for details.
"""
# TODO(John Sirois): Fixup TargetDefinitionException - it has awkward Target base-class
# initialization requirements right now requiring this Target.__init__.
Target.__init__(self, name, exclusives=exclusives)
if source is None and entry_point is None:
raise TargetDefinitionException(
self,
'A python binary target must specify either source or entry_point.'
)
PythonTarget.__init__(
self,
name,
[] if source is None else [source],
compatibility=compatibility,
dependencies=dependencies,
exclusives=exclusives,
)
if not isinstance(platforms, (list, tuple)) and not isinstance(
platforms, Compatibility.string):
raise TargetDefinitionException(
self, 'platforms must be a list, tuple or string.')
self._entry_point = entry_point
self._inherit_path = bool(inherit_path)
self._zip_safe = bool(zip_safe)
self._always_write_cache = bool(always_write_cache)
self._repositories = maybe_list(repositories or [])
self._indices = maybe_list(indices or [])
self._ignore_errors = bool(ignore_errors)
self._platforms = tuple(maybe_list(platforms or []))
if source and entry_point:
entry_point_module = entry_point.split(':', 1)[0]
source_entry_point = self._translate_to_entry_point(
self.sources[0])
if entry_point_module != source_entry_point:
raise TargetDefinitionException(
self,
'Specified both source and entry_point but they do not agree: %s vs %s'
% (source_entry_point, entry_point_module))
