def configure_python(self, source_roots, test_roots, lib_roots):
self.py_sources.extend(SourceSet(get_buildroot(), root, None, False) for root in source_roots)
self.py_sources.extend(SourceSet(get_buildroot(), root, None, True) for root in test_roots)
for root in lib_roots:
for path in os.listdir(os.path.join(get_buildroot(), root)):
if os.path.isdir(os.path.join(get_buildroot(), root, path)) or path.endswith('.egg'):
self.py_libs.append(SourceSet(get_buildroot(), root, path, False))
def configure_python(self, source_roots, test_roots, lib_roots):
self.py_sources.extend(SourceSet(get_buildroot(), root, None, False) for root in source_roots)
self.py_sources.extend(SourceSet(get_buildroot(), root, None, True) for root in test_roots)
for root in lib_roots:
for path in os.listdir(os.path.join(get_buildroot(), root)):
if os.path.isdir(os.path.join(get_buildroot(), root, path)) or path.endswith('.egg'):
self.py_libs.append(SourceSet(get_buildroot(), root, path, False))
def _resolve_paths(self, rel_base, paths):
"""
Resolves paths relative to the given rel_base from the build root.
For example:
target: ~/workspace/src/java/com/twitter/common/base/BUILD
rel_base: src/resources
Resolves paths from:
~/workspace/src/resources/com/twitter/common/base
"""
# meta targets are composed of already-resolved paths
if not paths or self.is_meta:
return paths
def flatten_paths(*items):
"""Flattens one or more items into a list. If the item is iterable each of its items is
flattened. If an item is callable, it is called and the result is flattened. Otherwise the
atom is appended to the flattened list. These rules are applied recursively such that the
returned list will only contain non-iterable, non-callable atoms."""
flat = []
def flatmap(item):
if isinstance(item, Compatibility.string):
flat.append(item)
else:
try:
for i in iter(item):
flatmap(i)
except:
if callable(item):
flatmap(item())
else:
flat.append(item)
for item in items:
flatmap(item)
return flat
src_relpath = os.path.relpath(
self.address.buildfile.parent_path,
os.path.join(get_buildroot(), self.target_base))
resolve_basepath = os.path.join(get_buildroot(), rel_base, src_relpath)
with pushd(resolve_basepath):
return [
os.path.normpath(os.path.join(src_relpath, path))
for path in flatten_paths(paths)
]
def __init__(self, basedir, *types):
"""
:basedir The base directory to resolve sources relative to
:types The target types to register :basedir: as a source root for
"""
basepath = os.path.abspath(basedir or os.path.curdir)
if get_buildroot() != os.path.commonprefix((basepath, get_buildroot())):
raise ValueError(
"The supplied basedir %s is not a sub-path of the project root %s" % (basepath, get_buildroot())
)
self.basedir = os.path.relpath(basepath, get_buildroot())
self.types = types
SourceRoot._register(self)
def _resolve_paths(self, rel_base, paths):
"""Resolves paths relative to the given rel_base from the build root.
For example:
target: ~/workspace/src/java/com/twitter/common/base/BUILD
rel_base: src/resources
Resolves paths from:
~/workspace/src/resources/com/twitter/common/base
"""
if not paths:
return []
def flatten_paths(*items):
"""Flattens one or more items into a list.
If the item is iterable each of its items is flattened. If an item is callable, it is called
and the result is flattened. Otherwise the atom is appended to the flattened list. These
rules are applied recursively such that the returned list will only contain non-iterable,
non-callable atoms.
"""
flat = []
def flatmap(item):
if isinstance(item, Compatibility.string):
flat.append(item)
else:
try:
for i in iter(item):
flatmap(i)
except TypeError:
if callable(item):
flatmap(item())
else:
flat.append(item)
for item in items:
flatmap(item)
return flat
src_relpath = os.path.relpath(
self.address.buildfile.parent_path, os.path.join(get_buildroot(), self.target_base)
)
resolve_basepath = os.path.join(get_buildroot(), rel_base, src_relpath)
with pushd(resolve_basepath):
return [os.path.normpath(os.path.join(src_relpath, path)) for path in flatten_paths(paths)]
def load(configpath=os.path.join(get_buildroot(), 'pants.ini'), defaults=None):
"""
Loads a Config from the given path, by default the path to the pants.ini file in the current
build root directory. Any defaults supplied will act as if specified in the loaded config
file's DEFAULT section. The 'buildroot', invoking 'user' and invoking user's 'homedir' are
automatically defaulted.
"""
standard_defaults = dict(
buildroot=get_buildroot(),
homedir=os.path.expanduser('~'),
user=getpass.getuser()
)
if defaults:
standard_defaults.update(defaults)
return Config(configpath, defaults=standard_defaults)
def __init__(self, basedir, *types):
"""
:basedir The base directory to resolve sources relative to
:types The target types to register :basedir: as a source root for
"""
basepath = os.path.abspath(basedir or os.path.curdir)
if get_buildroot() != os.path.commonprefix(
(basepath, get_buildroot())):
raise ValueError(
'The supplied basedir %s is not a sub-path of the project root %s'
% (basepath, get_buildroot()))
self.basedir = os.path.relpath(basepath, get_buildroot())
self.types = types
SourceRoot._register(self)
def findclasses(self, targets):
"""
Returns a mapping from a target to its source to classes mapping.
For example:
dependencies = Dependencies(outdir, depfile)
mapping = dependencies.findclasses(targets)
for target, src_to_classes in mapping.items():
for source, classes in src_to_classes.items():
print('source: %s produces classes: %s' % (
os.path.join(target.target_base, source),
[os.path.join(outdir, cls) for cls in classes]
))
"""
sources = set()
target_by_source = dict()
for target in targets:
for source in target.sources:
src = os.path.normpath(os.path.join(target.target_base, source))
target_by_source[src] = target
sources.add(src)
classes_by_target_by_source = defaultdict(lambda: defaultdict(set))
if os.path.exists(self.depfile):
with open(self.depfile, 'r') as deps:
for dep in deps.readlines():
src, cls = dep.strip().split('->')
sourcefile = os.path.relpath(os.path.join(self.outputdir, src.strip()), get_buildroot())
if sourcefile in sources:
classfile = os.path.relpath(os.path.join(self.outputdir, cls.strip()), self.outputdir)
target = target_by_source[sourcefile]
relsrc = os.path.relpath(sourcefile, target.target_base)
classes_by_target_by_source[target][relsrc].add(classfile)
return classes_by_target_by_source
def split(self, splits, catchall=False):
buildroot = get_buildroot()
src_to_split_idx = {}
for i, split in enumerate(splits):
for s in split:
src_to_split_idx[s if os.path.isabs(s) else os.path.join(buildroot, s)] = i
num_outputs = len(splits) + 1 if catchall else len(splits)
catchall_idx = len(splits) if catchall else -1
split_pcd_entries = []
split_src_to_deps = []
for _ in xrange(0, num_outputs):
split_pcd_entries.append([])
split_src_to_deps.append({})
for pcd_entry in self.pcd_entries:
split_idx = src_to_split_idx.get(pcd_entry[1], catchall_idx)
if split_idx != -1:
split_pcd_entries[split_idx].append(pcd_entry)
for src, deps in self.src_to_deps.items():
split_idx = src_to_split_idx.get(src, catchall_idx)
if split_idx != -1:
split_src_to_deps[split_idx][src] = deps
return [JMakeAnalysis(x, y) for x, y in zip(split_pcd_entries, split_src_to_deps)]
def _run():
root_dir = get_buildroot()
version = get_version()
if not os.path.exists(root_dir):
_exit_and_fail('PANTS_BUILD_ROOT does not point to a valid path: %s' % root_dir)
if len(sys.argv) < 2 or (len(sys.argv) == 2 and sys.argv[1] in _HELP_ALIASES):
_help(version, root_dir)
command_class, command_args = _parse_command(root_dir, sys.argv[1:])
parser = optparse.OptionParser(version = '%%prog %s' % version)
RcFile.install_disable_rc_option(parser)
parser.add_option(_LOG_EXIT_OPTION, action = 'store_true', dest = 'log_exit',
default = False, help = 'Log an exit message on success or failure')
command = command_class(root_dir, parser, command_args)
if command.serialized():
def onwait(pid):
print('Waiting on pants process %s to complete' % _process_info(pid), file=sys.stderr)
return True
runfile = os.path.join(root_dir, '.pants.run')
lock = Lock.acquire(runfile, onwait=onwait)
else:
lock = Lock.unlocked()
try:
result = command.run(lock)
_do_exit(result)
except KeyboardInterrupt:
command.cleanup()
raise
finally:
lock.release()
def make_build_properties(cls):
pi = PythonInterpreter()
base_info = {
'class': pi.identity().interpreter,
'version': pi.identity().version,
'platform': get_platform(),
}
try:
from twitter.pants import get_buildroot, get_scm
buildroot = get_buildroot()
scm = get_scm()
now = localtime()
if scm:
revision = scm.commit_id
tag = scm.tag_name or 'none'
branchname = scm.branch_name or revision
else:
revision = 'unknown'
tag = 'none'
branchname = 'unknown'
base_info.update({
'date': strftime('%A %b %d, %Y', now),
'time': strftime('%H:%M:%S', now),
'timestamp': strftime('%m.%d.%Y %H:%M', now),
'branch': branchname,
'tag': tag,
'sha': revision,
'user': getpass.getuser(),
'machine': socket.gethostname(),
'path': buildroot
})
except ImportError:
pass
return base_info
def get_port_and_pidfile(context):
port = context.options.port or context.config.getint('reporting', 'reporting_port')
# We don't put the pidfile in .pants.d, because we want to find it even after a clean.
# TODO: Fold pants.run and other pidfiles into here. Generalize the pidfile idiom into
# some central library.
pidfile = os.path.join(get_buildroot(), '.pids', 'port_%d.pid' % port)
return port, pidfile
def extract_target(java_targets, is_classpath):
primary_target = InternalTarget.sort_targets(java_targets)[0]
def create_target():
internal_deps, jar_deps = _extract_target(java_targets, is_classpath)
# TODO(John Sirois): make an empty source set work in ant/compile.xml
sources = [ '__no_source__' ]
all_deps = OrderedSet()
all_deps.update(internal_deps)
all_deps.update(jar_deps)
if is_java(primary_target):
return JavaLibrary('ide',
sources,
dependencies = all_deps,
is_meta = True)
elif is_scala(primary_target):
return ScalaLibrary('ide',
sources,
dependencies = all_deps,
is_meta = True)
else:
raise TypeError("Cannot generate IDE configuration for targets: %s" % java_targets)
buildfile = BuildFile(get_buildroot(), primary_target.target_base, must_exist=False)
return ParseContext(buildfile).do_in_context(create_target)
def _runjava_common(self, runjava, main, classpath=None, opts=None, args=None, jvmargs=None):
cp = (self._classpath or []) + (classpath or [])
if self._daemon:
nailgun = self._get_nailgun_client()
def call_nailgun(main_class, *args):
if self.dry_run:
print('********** NailgunClient dry run: %s %s' % (main_class, ' '.join(args)))
return 0
else:
return nailgun(main_class, *args)
try:
if cp:
call_nailgun('ng-cp', *[os.path.relpath(jar, get_buildroot()) for jar in cp])
opts_args = []
if opts:
opts_args.extend(opts)
if args:
opts_args.extend(args)
return call_nailgun(main, *opts_args)
except NailgunError as e:
self._ng_shutdown()
raise e
else:
ret = runjava(main=main, classpath=cp, opts=opts, args=args, jvmargs=jvmargs,
dryrun=self.dry_run)
if self.dry_run:
print('********** Direct Java dry run: %s' % ret)
return 0
else:
return ret
def make_build_properties(cls):
pi = PythonInterpreter()
base_info = {
'class': pi.identity().interpreter,
'version': pi.identity().version,
'platform': get_platform(),
}
try:
from twitter.pants import get_buildroot, get_scm
buildroot = get_buildroot()
scm = get_scm()
now = localtime()
revision = scm.commit_id
tag = scm.tag_name or 'none'
branchname = scm.branch_name or revision
base_info.update({
'date': strftime('%A %b %d, %Y', now),
'time': strftime('%H:%M:%S', now),
'timestamp': strftime('%m.%d.%Y %H:%M', now),
'branch': branchname,
'tag': tag,
'sha': revision,
'user': getpass.getuser(),
'machine': socket.gethostname(),
'path': buildroot
})
except ImportError:
pass
return base_info
def __init__(self, configparser, configpath):
# Base Config
self.configparser = configparser
with open(configpath) as ini:
self.configparser.readfp(ini, filename=configpath)
self.file = configpath
# Overrides
#
# This feature allows a second configuration file which will override
# pants.ini to be specified. The file is currently specified via an env
# variable because the cmd line flags are parsed after config is loaded.
#
# The main use of the extra file is to have different settings based on
# the environment. For example, the setting used to compile or locations
# of caches might be different between a developer's local environment
# and the environment used to build and publish artifacts (e.g. Jenkins)
#
# The files cannot reference each other's values, so make sure each one is
# internally consistent
self.overrides_path = os.environ.get('PANTS_CONFIG_OVERRIDE')
self.overrides_parser = None
if self.overrides_path is not None:
self.overrides_path = os.path.join(get_buildroot(), self.overrides_path)
self.overrides_parser = Config.create_parser()
with open(self.overrides_path) as o_ini:
self.overrides_parser.readfp(o_ini, filename=self.overrides_path)
def _spawn_nailgun_server(self):
log.info('No ng server found, spawning...')
with _safe_open(self._ng_out, 'w'):
pass # truncate
pid = os.fork()
if pid != 0:
# In the parent tine - block on ng being up for connections
return self._await_nailgun_server()
os.setsid()
in_fd = open('/dev/null', 'w')
out_fd = safe_open(self._ng_out, 'w')
err_fd = safe_open(self._ng_err, 'w')
args = ['java']
if self._ng_server_args:
args.extend(self._ng_server_args)
args.append(NailgunTask.PANTS_NG_ARG)
args.append(NailgunTask.create_pidfile_arg(self._pidfile))
args.extend(['-jar', self._nailgun_jar, ':0'])
log.debug('Executing: %s' % ' '.join(args))
with binary_utils.safe_classpath(logger=log.warn):
process = subprocess.Popen(args,
stdin=in_fd,
stdout=out_fd,
stderr=err_fd,
close_fds=True,
cwd=get_buildroot())
with _safe_open(self._pidfile, 'w') as pidfile:
pidfile.write('%d' % process.pid)
log.debug('Spawned ng server @ %d' % process.pid)
sys.exit(0)
def test_depeendees_type(self):
self._set_up_mocks(PythonTests, ["%s/tests" % get_buildroot()])
self.assert_console_output(
'tests/d/BUILD:d',
args=['--test-type=python_tests'],
targets=[self.target('common/d')]
)
def _owning_targets(self, file):
for build_file in self._candidate_owners(file):
is_build_file = (build_file.full_path == os.path.join(get_buildroot(), file))
for address in Target.get_all_addresses(build_file):
target = Target.get(address)
if target and (is_build_file or (has_sources(target) and self._owns(target, file))):
yield target
def _owning_targets(self, file):
for build_file in self._candidate_owners(file):
is_build_file = (build_file.full_path == os.path.join(get_buildroot(), file))
for address in Target.get_all_addresses(build_file):
target = Target.get(address)
if target and (is_build_file or (has_sources(target) and self._owns(target, file))):
yield target
def create_binary(self, binary):
import platform
safe_mkdir(self.outdir)
jarmap = self.context.products.get('jars')
binary_jarname = '%s.jar' % binary.basename
binaryjarpath = os.path.join(self.outdir, binary_jarname)
self.context.log.info('creating %s' % os.path.relpath(binaryjarpath, get_buildroot()))
with open_jar(binaryjarpath, 'w', compression=self.compression, allowZip64=self.zip64) as jar:
def add_jars(target):
generated = jarmap.get(target)
if generated:
for basedir, jars in generated.items():
for internaljar in jars:
self.dump(os.path.join(basedir, internaljar), jar)
binary.walk(add_jars, is_internal)
if self.deployjar:
for basedir, externaljar in self.list_jar_dependencies(binary):
self.dump(os.path.join(basedir, externaljar), jar)
manifest = Manifest()
manifest.addentry(Manifest.MANIFEST_VERSION, '1.0')
manifest.addentry(
Manifest.CREATED_BY,
'python %s pants %s (Twitter, Inc.)' % (platform.python_version(), get_version())
)
main = binary.main or '*** java -jar not supported, please use -cp and pick a main ***'
manifest.addentry(Manifest.MAIN_CLASS, main)
jar.writestr(Manifest.PATH, manifest.contents())
jarmap.add(binary, self.outdir, [binary_jarname])
def execute(self, targets):
if not self.skip:
args = []
if self.tests:
args.append('--classes')
args.append(','.join(self.tests))
else:
tests = self.calculate_tests(targets)
if tests:
args.append('--specs-files=%s' % ','.join(tests))
if args:
if self.color:
args.append('--color')
classpath = profile_classpath(self.profile)
classpath.extend(os.path.join(get_buildroot(), path)
for path in ('src/resources', 'tests/resources'))
with self.context.state('classpath', []) as cp:
classpath.extend(jar for conf, jar in cp if conf in self.confs)
result = runjava(
jvmargs=self.java_args,
classpath=classpath,
main='run' if self.tests else 'com.twitter.common.testing.ExplicitSpecsRunnerMain',
args=args
)
if result != 0:
raise TaskError()
def create_target(category, target_name, target_index, targets):
def name(name):
return "%s-%s-%d" % (target_name, name, target_index)
# TODO(John Sirois): JavaLibrary and ScalaLibrary can float here between src/ and tests/ - add
# ant build support to allow the same treatment for JavaThriftLibrary and JavaProtobufLibrary
# so that tests can house test IDL in tests/
target_type, base = category
def create():
if target_type == JavaProtobufLibrary:
return _aggregate(JavaProtobufLibrary, name('protobuf'), targets, buildflags=buildflags)
elif target_type == JavaThriftLibrary:
return _aggregate(JavaThriftLibrary, name('thrift'), targets, buildflags=buildflags)
elif target_type == AnnotationProcessor:
return _aggregate(AnnotationProcessor, name('apt'), targets)
elif target_type == JavaLibrary:
return _aggregate(JavaLibrary, name('java'), targets, deployjar, buildflags)
elif target_type == ScalaLibrary:
return _aggregate(ScalaLibrary, name('scala'), targets, deployjar, buildflags)
elif target_type == JavaTests:
return _aggregate(JavaTests, name('java-tests'), targets, buildflags=buildflags)
elif target_type == ScalaTests:
return _aggregate(ScalaTests, name('scala-tests'), targets, buildflags=buildflags)
else:
raise Exception("Cannot aggregate targets of type: %s" % target_type)
return ParseContext(BuildFile(get_buildroot(), base, must_exist=False)).do_in_context(create)
def configure_project(self, targets, checkstyle_suppression_files, debug_port,
scala_compiler_profile):
jvm_targets = extract_jvm_targets(targets)
if self.intransitive:
jvm_targets = set(self.context.target_roots).intersection(jvm_targets)
project = Project(self.project_name,
self.python,
self.skip_java,
self.skip_scala,
get_buildroot(),
checkstyle_suppression_files,
debug_port,
jvm_targets,
not self.intransitive,
self.context.new_workunit)
if self.python:
python_source_paths = self.context.config.getlist('ide', 'python_source_paths', default=[])
python_test_paths = self.context.config.getlist('ide', 'python_test_paths', default=[])
python_lib_paths = self.context.config.getlist('ide', 'python_lib_paths', default=[])
project.configure_python(python_source_paths, python_test_paths, python_lib_paths)
extra_source_paths = self.context.config.getlist('ide', 'extra_jvm_source_paths', default=[])
extra_test_paths = self.context.config.getlist('ide', 'extra_jvm_test_paths', default=[])
all_targets = project.configure_jvm(
scala_compiler_profile,
extra_source_paths,
extra_test_paths
)
return all_targets, project
def __init__(self, context, classpath=None, workdir=None):
Task.__init__(self, context)
self._classpath = classpath
self._nailgun_bootstrap_key = 'nailgun'
nailgun_bootstrap_tools = context.config.getlist('nailgun', 'bootstrap-tools',
default=[':nailgun-server'])
self._bootstrap_utils.register_jvm_build_tools(self._nailgun_bootstrap_key, nailgun_bootstrap_tools)
self._ng_server_args = context.config.getlist('nailgun', 'args')
self._daemon = context.options.nailgun_daemon
workdir = workdir or context.config.get('nailgun', 'workdir')
# Allows us to identify the nailgun process by its cmd-line.
self._identifier_arg = '-Dpants.ng.identifier=%s' % os.path.relpath(workdir, get_buildroot())
self._current_pidport = None
self._ng_out = os.path.join(workdir, 'stdout')
self._ng_err = os.path.join(workdir, 'stderr')
# Prevent concurrency issues when starting up a nailgun.
self._spawn_lock = threading.Lock()
def __init__(self, context, **kwargs):
super(ListTargets, self).__init__(context, **kwargs)
self._provides = context.options.list_provides
self._provides_columns = context.options.list_provides_columns
self._documented = context.options.list_documented
self._root_dir = get_buildroot()
def __init__(self, context, workdir=None):
NailgunTask.__init__(self, context, workdir=context.config.get('scala-compile', 'nailgun_dir'))
# Set up the zinc utils.
color = context.options.scala_compile_color or \
context.config.getbool('scala-compile', 'color', default=True)
self._zinc_utils = ZincUtils(context=context, java_runner=self.runjava, color=color)
# The rough number of source files to build in each compiler pass.
self._partition_size_hint = \
context.options.scala_compile_partition_size_hint \
if context.options.scala_compile_partition_size_hint != -1 else \
context.config.getint('scala-compile', 'partition_size_hint')
# Set up dep checking if needed.
if context.options.scala_check_missing_deps:
JvmDependencyCache.init_product_requirements(self)
# Various output directories.
self._buildroot = get_buildroot()
workdir = context.config.get('scala-compile', 'workdir') if workdir is None else workdir
self._resources_dir = os.path.join(workdir, 'resources')
self._artifact_factory = ZincArtifactFactory(workdir, self.context, self._zinc_utils)
# The ivy confs for which we're building.
self._confs = context.config.getlist('scala-compile', 'confs')
# The artifact cache to read from/write to.
artifact_cache_spec = context.config.getlist('scala-compile', 'artifact_caches')
self.setup_artifact_cache(artifact_cache_spec)
def __init__(self, context, **kwargs):
super(ListTargets, self).__init__(context, **kwargs)
self._provides = context.options.list_provides
self._provides_columns = context.options.list_provides_columns
self._documented = context.options.list_documented
self._root_dir = get_buildroot()
def find(target):
"""
Finds the source root for the given target target. If none is registered, the parent
directory of the target's BUILD file is returned.
"""
target_path = os.path.relpath(target.address.buildfile.parent_path, get_buildroot())
def _find():
for typ in target.__class__.mro():
for root in SourceRoot._ROOTS.get(typ, ()):
if target_path.startswith(root):
return root
# Try already registered roots
root = _find()
if root:
return root
# Fall back to searching the ancestor path for a root
for buildfile in reversed(target.address.buildfile.ancestors()):
if buildfile not in SourceRoot._SEARCHED:
SourceRoot._SEARCHED.add(buildfile)
ParseContext(buildfile).parse()
root = _find()
if root:
return root
# Finally, resolve files relative to the BUILD file parent dir as the target base
return target_path
def parse_jarcoordinate(coordinate):
components = coordinate.split('#', 1)
if len(components) == 2:
org, name = components
return org, name
else:
try:
address = Address.parse(get_buildroot(), coordinate)
try:
target = Target.get(address)
if not target:
siblings = Target.get_all_addresses(
address.buildfile)
prompt = 'did you mean' if len(
siblings
) == 1 else 'maybe you meant one of these'
raise TaskError('%s => %s?:\n %s' %
(address, prompt, '\n '.join(
str(a) for a in siblings)))
if not is_exported(target):
raise TaskError('%s is not an exported target' %
coordinate)
return target.provides.org, target.provides.name
except (ImportError, SyntaxError, TypeError):
raise TaskError('Failed to parse %s' %
address.buildfile.relpath)
except IOError:
raise TaskError('No BUILD file could be found at %s' %
coordinate)
def _run():
root_dir = get_buildroot()
version = get_version()
if not os.path.exists(root_dir):
_exit_and_fail('PANTS_BUILD_ROOT does not point to a valid path: %s' % root_dir)
if len(sys.argv) < 2 or (len(sys.argv) == 2 and sys.argv[1] in _HELP_ALIASES):
_help(version, root_dir)
command_class, command_args = _parse_command(root_dir, sys.argv[1:])
parser = optparse.OptionParser(version = '%%prog %s' % version)
RcFile.install_disable_rc_option(parser)
parser.add_option(_LOG_EXIT_OPTION, action = 'store_true', dest = 'log_exit',
default = False, help = 'Log an exit message on success or failure')
command = command_class(root_dir, parser, command_args)
if command.serialized():
def onwait(pid):
print('Waiting on pants process %s to complete' % _process_info(pid), file=sys.stderr)
return True
runfile = os.path.join(root_dir, '.pants.run')
lock = Lock.acquire(runfile, onwait=onwait)
else:
lock = Lock.unlocked()
try:
result = command.run(lock)
_do_exit(result)
finally:
lock.release()
def __init__(self, context, nailgun_task, color, bootstrap_utils):
self.context = context
self._nailgun_task = nailgun_task # We run zinc on this task's behalf.
self._color = color
self._bootstrap_utils = bootstrap_utils
self._pants_home = get_buildroot()
# The target scala version.
self._compile_bootstrap_key = 'scalac'
compile_bootstrap_tools = context.config.getlist('scala-compile', 'compile-bootstrap-tools',
default=[':scala-compile-2.9.2'])
self._bootstrap_utils.register_jvm_build_tools(self._compile_bootstrap_key, compile_bootstrap_tools)
# The zinc version (and the scala version it needs, which may differ from the target version).
self._zinc_bootstrap_key = 'zinc'
zinc_bootstrap_tools = context.config.getlist('scala-compile', 'zinc-bootstrap-tools', default=[':zinc'])
self._bootstrap_utils.register_jvm_build_tools(self._zinc_bootstrap_key, zinc_bootstrap_tools)
# Compiler plugins.
plugins_bootstrap_tools = context.config.getlist('scala-compile', 'scalac-plugin-bootstrap-tools',
default=[])
if plugins_bootstrap_tools:
self._plugins_bootstrap_key = 'plugins'
self._bootstrap_utils.register_jvm_build_tools(self._plugins_bootstrap_key, plugins_bootstrap_tools)
else:
self._plugins_bootstrap_key = None
self._main = context.config.get('scala-compile', 'main')
self._jvm_args = context.config.getlist('scala-compile', 'jvm_args')
# For localizing/relativizing analysis files.
self._java_home = context.java_home
self._ivy_home = context.ivy_home
def get_port_and_pidfile(context):
port = context.options.port or context.config.getint('reporting', 'reporting_port')
# We don't put the pidfile in .pants.d, because we want to find it even after a clean.
# TODO: Fold pants.run and other pidfiles into here. Generalize the pidfile idiom into
# some central library.
pidfile = os.path.join(get_buildroot(), '.pids', 'port_%d.pid' % port)
return port, pidfile
def configure_project(self, targets, checkstyle_suppression_files,
debug_port):
jvm_targets = Target.extract_jvm_targets(targets)
if self.intransitive:
jvm_targets = set(
self.context.target_roots).intersection(jvm_targets)
project = Project(self.project_name,
self.python, self.skip_java, self.skip_scala,
get_buildroot(), checkstyle_suppression_files,
debug_port, jvm_targets, not self.intransitive,
self.context.new_workunit)
if self.python:
python_source_paths = self.context.config.getlist(
'ide', 'python_source_paths', default=[])
python_test_paths = self.context.config.getlist(
'ide', 'python_test_paths', default=[])
python_lib_paths = self.context.config.getlist('ide',
'python_lib_paths',
default=[])
project.configure_python(python_source_paths, python_test_paths,
python_lib_paths)
extra_source_paths = self.context.config.getlist(
'ide', 'extra_jvm_source_paths', default=[])
extra_test_paths = self.context.config.getlist('ide',
'extra_jvm_test_paths',
default=[])
all_targets = project.configure_jvm(extra_source_paths,
extra_test_paths)
return all_targets, project
def find(target):
"""
Finds the source root for the given target target. If none is registered, the parent
directory of the target's BUILD file is returned.
"""
target_path = os.path.relpath(target.address.buildfile.parent_path,
get_buildroot())
def _find():
for typ in target.__class__.mro():
for root in SourceRoot._ROOTS.get(typ, ()):
if target_path.startswith(root):
return root
# Try already registered roots
root = _find()
if root:
return root
# Fall back to searching the ancestor path for a root
for buildfile in reversed(target.address.buildfile.ancestors()):
if buildfile not in SourceRoot._SEARCHED:
SourceRoot._SEARCHED.add(buildfile)
ParseContext(buildfile).parse()
root = _find()
if root:
return root
# Finally, resolve files relative to the BUILD file parent dir as the target base
return target_path
def execute(self, targets):
gentargets = [t for t in targets if self.is_gentarget(t)]
capabilities = self.genlangs() # lang_name => predicate
gentargets_by_dependee = self.context.dependents(
on_predicate=self.is_gentarget,
from_predicate=lambda t: not self.is_gentarget(t)
)
dependees_by_gentarget = defaultdict(set)
for dependee, tgts in gentargets_by_dependee.items():
for gentarget in tgts:
dependees_by_gentarget[gentarget].add(dependee)
def find_gentargets(predicate):
tgts = set()
for dependee in gentargets_by_dependee.keys():
if predicate(dependee):
for tgt in gentargets_by_dependee.pop(dependee):
tgt.walk(tgts.add, self.is_gentarget)
return tgts.intersection(set(gentargets))
gentargets_bylang = {}
for lang, predicate in capabilities.items():
gentargets_bylang[lang] = gentargets if self.is_forced(lang) else find_gentargets(predicate)
if gentargets_by_dependee:
self.context.log.warn('Left with unexpected unconsumed gen targets:\n\t%s' % '\n\t'.join(
'%s -> %s' % (dependee, gentargets)
for dependee, gentargets in gentargets_by_dependee.items()
))
with self.invalidated(gentargets, invalidate_dependents=True) as invalidation_check:
for vt in invalidation_check.invalid_vts_partitioned:
invalid_targets = set(vt.targets)
artifact_files = []
for lang, tgts in gentargets_bylang.items():
lang_invalid = invalid_targets.intersection(tgts)
if lang_invalid:
artifact_files.extend(self.genlang(lang, lang_invalid) or [])
if self._artifact_cache and self.context.options.write_to_artifact_cache and artifact_files:
self.update_artifact_cache(vt, artifact_files)
# Link synthetic targets for all in-play gen targets
for lang, tgts in gentargets_bylang.items():
if tgts:
langtarget_by_gentarget = {}
for target in tgts:
langtarget_by_gentarget[target] = self.createtarget(
lang,
target,
dependees_by_gentarget.get(target, [])
)
langtarget_by_gentarget[target].add_label("synthetic")
genmap = self.context.products.get(lang)
for gentarget, langtarget in langtarget_by_gentarget.items():
genmap.add(gentarget, get_buildroot(), [langtarget])
# Transfer dependencies from gentarget to its synthetic counterpart.
for dep in self.getdependencies(gentarget):
if self.is_gentarget(dep): # Translate the dep to its synthetic counterpart.
self.updatedependencies(langtarget, langtarget_by_gentarget[dep])
else: # Depend directly on the dep.
self.updatedependencies(langtarget, dep)
def runjava(self, main, classpath=None, args=None, jvmargs=None):
"""
Runs the java main using the given classpath and args. If --no-ng-daemons is specified then
the java main is run in a freshly spawned subprocess, otherwise a persistent nailgun server
dedicated to this Task subclass is used to speed up amortized run times.
"""
cp = (self._classpath or []) + (classpath or [])
if self._daemon:
nailgun = self._get_nailgun_client()
def call_nailgun(main_class, *args):
if self.dry_run:
print('********** NailgunClient dry run: %s %s' % (main_class, ' '.join(args)))
return 0
else:
return nailgun(main_class, *args)
try:
if cp:
call_nailgun('ng-cp', *[os.path.relpath(jar, get_buildroot()) for jar in cp])
return call_nailgun(main, *args)
except NailgunError as e:
self._ng_shutdown()
raise e
else:
only_write_cmd_line_to = StringIO.StringIO() if self.dry_run else None
ret = binary_utils.runjava(main=main, classpath=cp, args=args, jvmargs=jvmargs,
only_write_cmd_line_to=only_write_cmd_line_to)
if only_write_cmd_line_to:
print('********** Direct Java dry run: %s' % only_write_cmd_line_to.getvalue())
only_write_cmd_line_to.close()
return ret
def findclasses(self, targets):
sources = set()
target_by_source = dict()
for target in targets:
for source in target.sources:
src = os.path.normpath(os.path.join(target.target_base, source))
target_by_source[src] = target
sources.add(src)
classes_by_target_by_source = defaultdict(lambda: defaultdict(set))
if os.path.exists(self.depfile):
with open(self.depfile, 'r') as deps:
section = 0
for dep in deps.readlines():
line = dep.strip()
if '-------' == line:
section += 1
elif ScalaCompiler._SECTIONS[section] == 'source_to_class':
src, cls = line.split('->')
sourcefile = os.path.relpath(os.path.join(self.outputdir, src.strip()), get_buildroot())
if sourcefile in sources:
classfile = os.path.relpath(os.path.join(self.outputdir, cls.strip()), self.outputdir)
target = target_by_source[sourcefile]
relsrc = os.path.relpath(sourcefile, target.target_base)
classes_by_target_by_source[target][relsrc].add(classfile)
return classes_by_target_by_source
def __init__(self, context, java_runner, color):
self._context = context
self._java_runner = java_runner
self._color = color
self._pants_home = get_buildroot()
self._compile_profile = context.config.get(
'scala-compile', 'compile-profile') # The target scala version.
self._zinc_profile = context.config.get('scala-compile',
'zinc-profile')
self._plugins_profile = context.config.get('scala-compile',
'scalac-plugins-profile')
self._main = context.config.get('scala-compile', 'main')
self._scalac_args = context.config.getlist('scala-compile', 'args')
self._jvm_args = context.config.getlist('scala-compile', 'jvm_args')
if context.options.scala_compile_warnings:
self._scalac_args.extend(
context.config.getlist('scala-compile', 'warning_args'))
else:
self._scalac_args.extend(
context.config.getlist('scala-compile', 'no_warning_args'))
def classpath_for_profile(profile):
return profile_classpath(profile,
java_runner=self._java_runner,
config=self._context.config)
self._zinc_classpath = classpath_for_profile(self._zinc_profile)
self._compiler_classpath = classpath_for_profile(self._compile_profile)
self._plugin_jars = (classpath_for_profile(self._plugins_profile)
if self._plugins_profile else [])
zinc_jars = ZincUtils.identify_zinc_jars(self._compiler_classpath,
self._zinc_classpath)
self._zinc_jar_args = []
for (name, jarpath) in zinc_jars.items(
): # The zinc jar names are also the flag names.
self._zinc_jar_args.extend(['-%s' % name, jarpath])
# Allow multiple flags and also comma-separated values in a single flag.
plugin_names = [p for val in context.options.plugins for p in val.split(',')]\
if context.options.plugins is not None\
else context.config.getlist('scala-compile', 'scalac-plugins', default=[])
plugin_args = context.config.getdict('scala-compile',
'scalac-plugin-args',
default={})
active_plugins = self.find_plugins(plugin_names)
for name, jar in active_plugins.items():
self._scalac_args.append('-Xplugin:%s' % jar)
for arg in plugin_args.get(name, []):
self._scalac_args.append('-P:%s:%s' % (name, arg))
# For localizing/relativizing analysis files.
self._java_home = os.path.dirname(find_java_home())
self._ivy_home = context.config.get('ivy', 'cache_dir')
def process(self, outdir, base, source, standalone, url_builder, get_config, css=None):
def parse_url(spec):
match = MarkdownToHtml.PANTS_LINK.match(spec)
if match:
page = Target.get(Address.parse(get_buildroot(), match.group(1)))
if not page:
raise TaskError('Invalid link %s' % match.group(1))
alias, url = url_builder(page, config=get_config(page))
return alias, url
else:
return spec, spec
def build_url(label):
components = label.split('|', 1)
if len(components) == 1:
return parse_url(label.strip())
else:
alias, link = components
_, url = parse_url(link.strip())
return alias, url
wikilinks = WikilinksExtension(build_url)
path, ext = os.path.splitext(source)
with safe_open(os.path.join(outdir, path + '.html'), 'w') as output:
with open(os.path.join(get_buildroot(), base, source), 'r') as input:
md_html = markdown.markdown(
input.read(),
extensions=['codehilite(guess_lang=False)', 'extra', 'tables', 'toc', wikilinks],
)
if standalone:
if css:
css_relpath = os.path.relpath(css, outdir)
out_relpath = os.path.dirname(source)
link_relpath = os.path.relpath(css_relpath, out_relpath)
css = '<link rel="stylesheet" type="text/css" href="%s"/>' % link_relpath
html = textwrap.dedent('''
<html>
<head>
%s
</head>
<body>
<!-- generated by pants! -->
%s
</body>
</html>
''').strip() % (css or '', md_html)
output.write(html)
else:
if css:
with safe_open(css) as fd:
output.write(textwrap.dedent('''
<style type="text/css">
%s
</style>
''').strip() % fd.read())
output.write('\n')
output.write(md_html)
return output.name
def process(self, outdir, base, source, standalone, url_builder, get_config, css=None):
def parse_url(spec):
match = MarkdownToHtml.PANTS_LINK.match(spec)
if match:
page = Target.get(Address.parse(get_buildroot(), match.group(1)))
if not page:
raise TaskError('Invalid link %s' % match.group(1))
alias, url = url_builder(page, config=get_config(page))
return alias, url
else:
return spec, spec
def build_url(label):
components = label.split('|', 1)
if len(components) == 1:
return parse_url(label.strip())
else:
alias, link = components
_, url = parse_url(link.strip())
return alias, url
wikilinks = WikilinksExtension(build_url)
path, ext = os.path.splitext(source)
with safe_open(os.path.join(outdir, path + '.html'), 'w') as output:
with open(os.path.join(get_buildroot(), base, source), 'r') as input:
md_html = markdown.markdown(
input.read(),
extensions=['codehilite', 'extra', 'toc', wikilinks]
)
if standalone:
if css:
css_relpath = os.path.relpath(css, outdir)
out_relpath = os.path.dirname(source)
link_relpath = os.path.relpath(css_relpath, out_relpath)
css = '<link rel="stylesheet" type="text/css" href="%s"/>' % link_relpath
html = textwrap.dedent('''
<html>
<head>
%s
</head>
<body>
<!-- generated by pants! -->
%s
</body>
</html>
''').strip() % (css or '', md_html)
output.write(html)
else:
if css:
with safe_open(css) as fd:
output.write(textwrap.dedent('''
<style type="text/css">
%s
</style>
''').strip() % fd.read())
output.write('\n')
output.write(md_html)
return output.name
def add_basic_info(self, run_id, timestamp):
"""A helper function to add basic build info."""
datetime = time.strftime('%A %b %d, %Y %H:%M:%S', time.localtime(timestamp))
user = getpass.getuser()
machine = socket.gethostname()
path = get_buildroot()
self.add_infos(('id', run_id), ('timestamp', timestamp), ('datetime', datetime),
('user', user), ('machine', machine), ('path', path))
def add_resource_paths(predicate):
bases = set()
for target in self.context.targets():
if predicate(target):
if target.target_base not in bases:
sibling_resources_base = os.path.join(os.path.dirname(target.target_base), "resources")
classpath.append(os.path.join(get_buildroot(), sibling_resources_base))
bases.add(target.target_base)
def load(configpath=os.path.join(get_buildroot(), 'pants.ini'), defaults=None):
"""
Loads a Config from the given path, by default the path to the pants.ini file in the current
build root directory. Any defaults supplied will act as if specified in the loaded config
file's DEFAULT section. The 'buildroot', invoking 'user' and invoking user's 'homedir' are
automatically defaulted.
"""
return Config(Config.create_parser(defaults), configpath)
def _candidate_owners(self, file):
build_file = BuildFile(get_buildroot(), relpath=os.path.dirname(file), must_exist=False)
if build_file.exists():
yield build_file
for sibling in build_file.siblings():
yield sibling
for ancestor in build_file.ancestors():
yield ancestor
def _candidate_owners(self, file):
build_file = BuildFile(get_buildroot(), relpath=os.path.dirname(file), must_exist=False)
if build_file.exists():
yield build_file
for sibling in build_file.siblings():
yield sibling
for ancestor in build_file.ancestors():
yield ancestor
def _run():
version = get_version()
if len(sys.argv) == 2 and sys.argv[1] == _VERSION_OPTION:
_do_exit(version)
root_dir = get_buildroot()
if not os.path.exists(root_dir):
_exit_and_fail('PANTS_BUILD_ROOT does not point to a valid path: %s' %
root_dir)
if len(sys.argv) < 2 or (len(sys.argv) == 2
and sys.argv[1] in _HELP_ALIASES):
_help(version, root_dir)
command_class, command_args = _parse_command(root_dir, sys.argv[1:])
parser = optparse.OptionParser(version=version)
RcFile.install_disable_rc_option(parser)
parser.add_option(_LOG_EXIT_OPTION,
action='store_true',
default=False,
dest='log_exit',
help='Log an exit message on success or failure.')
config = Config.load()
run_tracker = RunTracker(config)
report = default_report(config, run_tracker)
run_tracker.start(report)
url = run_tracker.run_info.get_info('report_url')
run_tracker.log(Report.INFO, 'See a report at: %s' % url)
run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
try:
command = command_class(run_tracker, root_dir, parser, command_args)
if command.serialized():
def onwait(pid):
print('Waiting on pants process %s to complete' %
_process_info(pid),
file=sys.stderr)
return True
runfile = os.path.join(root_dir, '.pants.run')
lock = Lock.acquire(runfile, onwait=onwait)
else:
lock = Lock.unlocked()
try:
result = command.run(lock)
_do_exit(result)
except KeyboardInterrupt:
command.cleanup()
raise
finally:
lock.release()
finally:
run_tracker.end()
def execute(self, targets):
gentargets = [t for t in targets if self.is_gentarget(t)]
capabilities = self.genlangs() # lang_name => predicate
gentargets_by_dependee = self.context.dependants(
on_predicate=self.is_gentarget,
from_predicate=lambda t: not self.is_gentarget(t))
dependees_by_gentarget = defaultdict(set)
for dependee, tgts in gentargets_by_dependee.items():
for gentarget in tgts:
dependees_by_gentarget[gentarget].add(dependee)
def find_gentargets(predicate):
tgts = set()
for dependee in gentargets_by_dependee.keys():
if predicate(dependee):
for tgt in gentargets_by_dependee.pop(dependee):
tgt.walk(tgts.add, self.is_gentarget)
return tgts.intersection(set(gentargets))
gentargets_bylang = {}
for lang, predicate in capabilities.items():
gentargets_bylang[lang] = gentargets if self.is_forced(
lang) else find_gentargets(predicate)
if gentargets_by_dependee:
self.context.log.warn(
'Left with unexpected unconsumed gen targets:\n\t%s' %
'\n\t'.join('%s -> %s' % (dependee, gentargets) for dependee,
gentargets in gentargets_by_dependee.items()))
with self.invalidated(
gentargets, invalidate_dependants=True) as invalidation_check:
invalid_targets = set()
for vt in invalidation_check.invalid_vts:
invalid_targets.update(vt.targets)
for lang, tgts in gentargets_bylang.items():
lang_invalid = invalid_targets.intersection(tgts)
if lang_invalid:
self.genlang(lang, lang_invalid)
# Link synthetic targets for all in-play gen targets
for lang, tgts in gentargets_bylang.items():
if tgts:
langtarget_by_gentarget = {}
for target in tgts:
langtarget_by_gentarget[target] = self.createtarget(
lang, target, dependees_by_gentarget.get(target, []))
genmap = self.context.products.get(lang)
for gentarget, langtarget in langtarget_by_gentarget.items():
genmap.add(gentarget, get_buildroot(), [langtarget])
# Transfer dependencies from gentarget to its synthetic counterpart.
for dep in self.getdependencies(gentarget):
if self.is_gentarget(
dep
): # Translate the dep to its synthetic counterpart.
self.updatedependencies(
langtarget, langtarget_by_gentarget[dep])
else: # Depend directly on the dep.
self.updatedependencies(langtarget, dep)
def run_server(reporting_queue):
(port, pidfile) = get_port_and_pidfile(self.context)
def write_pidfile():
safe_mkdir(os.path.dirname(pidfile))
with open(pidfile, 'w') as outfile:
outfile.write(str(os.getpid()))
def report_launch():
reporting_queue.put(
'Launching server with pid %d at http://localhost:%d' %
(os.getpid(), port))
show_latest_run_msg()
def show_latest_run_msg():
url = 'http://localhost:%d/run/latest' % port
try:
from colors import magenta
url = magenta(url)
except ImportError:
pass
reporting_queue.put('Automatically see latest run at %s' % url)
def done_reporting():
reporting_queue.put(DONE)
try:
# We mustn't block in the child, because the multiprocessing module enforces that the
# parent either kills or joins to it. Instead we fork a grandchild that inherits the queue
# but is allowed to block indefinitely on the server loop.
if not os.fork():
# Child process.
info_dir = self.context.config.getdefault('info_dir')
template_dir = self.context.config.get(
'reporting', 'reports_template_dir')
assets_dir = self.context.config.get(
'reporting', 'reports_assets_dir')
settings = ReportingServer.Settings(
info_dir=info_dir,
template_dir=template_dir,
assets_dir=assets_dir,
root=get_buildroot(),
allowed_clients=self.context.options.allowed_clients)
server = ReportingServer(port, settings)
# Block forever here.
server.start(run_before_blocking=[
write_pidfile, report_launch, done_reporting
])
except socket.error, e:
if e.errno == errno.EADDRINUSE:
reporting_queue.put(
'Server already running at http://localhost:%d' % port)
show_latest_run_msg()
done_reporting()
return
else:
done_reporting()
raise
def _get_target(address):
try:
address = Address.parse(get_buildroot(), address, is_relative=False)
except IOError as e:
raise TaskError('Failed to parse address: %s: %s' % (address, e))
match = Target.get(address)
if not match:
raise TaskError('Invalid target address: %s' % address)
return match
def temp(basedir=None):
"""Activates a temporary parse context in the given basedir relative to the build root or else
in the build root dir itself if no basedir is specified."""
context = ParseContext(
BuildFile(get_buildroot(),
basedir or 'BUILD.temp',
must_exist=False))
with ParseContext.activate(context):
yield
def genlang(self, lang, targets):
bases, sources = calculate_compile_roots(targets, self.is_gentarget)
if lang == 'java':
gen = self.gen_java.gen
elif lang == 'python':
gen = self.gen_python.gen
else:
raise TaskError('Unrecognized thrift gen lang: %s' % lang)
args = [
self.thrift_binary,
'--gen',
gen,
'-recurse',
]
if self.strict:
args.append('-strict')
if self.verbose:
args.append('-verbose')
for base in bases:
args.extend(('-I', base))
sessions = []
for source in sources:
self.context.log.info('Generating thrift for %s\n' % source)
# Create a unique session dir for this thrift root. Sources may be full paths but we only
# need the path relative to the build root to ensure uniqueness.
# TODO(John Sirois): file paths should be normalized early on and uniformly, fix the need to
# relpath here at all.
relsource = os.path.relpath(source, get_buildroot())
outdir = os.path.join(self.session_dir,
'.'.join(relsource.split(os.path.sep)))
safe_mkdir(outdir)
cmd = args[:]
cmd.extend(('-o', outdir))
cmd.append(source)
log.debug('Executing: %s' % ' '.join(cmd))
sessions.append(
self.ThriftSession(outdir, cmd, subprocess.Popen(cmd)))
result = 0
for session in sessions:
if result != 0:
session.process.kill()
else:
result = session.process.wait()
if result != 0:
self.context.log.error('Failed: %s' %
' '.join(session.cmd))
else:
_copytree(session.outdir, self.combined_dir)
if result != 0:
raise TaskError('thrift compile failed with exit code %d' % result)
def add_basic_info(self, run_id, timestamp):
"""Adds basic build info and returns a dict composed of just this added info."""
datetime = time.strftime('%A %b %d, %Y %H:%M:%S',
time.localtime(timestamp))
user = getpass.getuser()
machine = socket.gethostname()
path = get_buildroot()
return self.add_infos(('id', run_id), ('timestamp', timestamp),
('datetime', datetime), ('user', user),
('machine', machine), ('path', path))
def _get_resource_extensions(self, project):
resource_extensions = set()
resource_extensions.update(project.resource_extensions)
# TODO(John Sirois): make test resources 1st class in ant build and punch this through to pants
# model
for _, _, files in os.walk(os.path.join(get_buildroot(), 'tests', 'resources')):
resource_extensions.update(Project.extract_resource_extensions(files))
return resource_extensions
