def setUp(self):
super(JvmToolTaskTestBase, self).setUp()
# Use a synthetic subclass for proper isolation when bootstrapping within the test.
bootstrap_scope = 'bootstrap_scope'
self.bootstrap_task_type = self.synthesize_task_subtype(BootstrapJvmTools, bootstrap_scope)
JvmToolMixin.reset_registered_tools()
# Set some options:
# 1. Cap BootstrapJvmTools memory usage in tests. The Xmx was empirically arrived upon using
# -Xloggc and verifying no full gcs for a test using the full gamut of resolving a multi-jar
# tool, constructing a fat jar and then shading that fat jar.
#
# 2. Allow tests to read/write tool jars from the real artifact cache, so they don't
# each have to resolve and shade them every single time, which is a huge slowdown.
# Note that local artifact cache writes are atomic, so it's fine for multiple concurrent
# tests to write to it.
#
# Note that we don't have access to the invoking pants instance's options, so we assume that
# its artifact cache is in the standard location. If it isn't, worst case the tests will
# populate a second cache at the standard location, which is no big deal.
# TODO: We really need a straightforward way for pants's own tests to get to the enclosing
# pants instance's options values.
artifact_caches = [os.path.join(get_pants_cachedir(), 'artifact_cache')]
self.set_options_for_scope(bootstrap_scope, jvm_options=['-Xmx128m'])
self.set_options_for_scope('cache.{}'.format(bootstrap_scope),
read_from=artifact_caches,
write_to=artifact_caches)
# Tool option defaults currently point to targets in the real BUILD.tools, so we copy it
# into our test workspace.
shutil.copy(os.path.join(self.real_build_root, 'BUILD.tools'), self.build_root)
Bootstrapper.reset_instance()
def execute(self):
context = self.context
if JvmToolMixin.get_registered_tools():
# Map of scope -> (map of key -> callback).
callback_product_map = (context.products.get_data(
'jvm_build_tools_classpath_callbacks') or defaultdict(dict))
# We leave a callback in the products map because we want these Ivy calls
# to be done lazily (they might never actually get executed) and we want
# to hit Task.invalidated (called in Task.ivy_resolve) on the instance of
# BootstrapJvmTools rather than the instance of whatever class requires
# the bootstrap tools. It would be awkward and possibly incorrect to call
# self.invalidated twice on a Task that does meaningful invalidation on its
# targets. -pl
for scope, key, main, custom_rules in JvmToolMixin.get_registered_tools(
):
option = key.replace('-', '_')
deplist = self.context.options.for_scope(scope)[option]
callback_product_map[scope][
key] = self.cached_bootstrap_classpath_callback(
key,
scope,
deplist,
main=main,
custom_rules=custom_rules)
context.products.safe_create_data(
'jvm_build_tools_classpath_callbacks',
lambda: callback_product_map)
def setUp(self):
# Ensure we get a read of the real pants.ini config
Config.reset_default_bootstrap_option_values()
real_config = Config.from_cache()
super(JvmToolTaskTestBase, self).setUp()
# Use a synthetic subclass for bootstrapping within the test, to isolate this from
# any bootstrapping the pants run executing the test might need.
self.bootstrap_task_type, bootstrap_scope = self.synthesize_task_subtype(BootstrapJvmTools)
JvmToolMixin.reset_registered_tools()
# Cap BootstrapJvmTools memory usage in tests. The Xmx was empirically arrived upon using
# -Xloggc and verifying no full gcs for a test using the full gamut of resolving a multi-jar
# tool, constructing a fat jar and then shading that fat jar.
self.set_options_for_scope(bootstrap_scope, jvm_options=['-Xmx128m'])
def link_or_copy(src, dest):
try:
os.link(src, dest)
except OSError as e:
if e.errno == errno.EXDEV:
shutil.copy(src, dest)
else:
raise e
def link(path, optional=False, force=False):
src = os.path.join(self.real_build_root, path)
if not optional or os.path.exists(src):
dest = os.path.join(self.build_root, path)
safe_mkdir(os.path.dirname(dest))
try:
link_or_copy(src, dest)
except OSError as e:
if force and e.errno == errno.EEXIST:
os.unlink(dest)
link_or_copy(src, dest)
else:
raise e
return dest
def link_tree(path, optional=False, force=False):
src = os.path.join(self.real_build_root, path)
if not optional or os.path.exists(src):
for abspath, dirs, files in safe_walk(src):
for f in files:
link(os.path.relpath(os.path.join(abspath, f), self.real_build_root), force=force)
# TODO(John Sirois): Find a way to do this cleanly
link('pants.ini', force=True)
# TODO(pl): Note that this pulls in a big chunk of the hairball to every test that
# depends on it, because BUILD contains source_roots that specify a variety of types
# from different backends.
link('BUILD', force=True)
link('BUILD.tools', force=True)
support_dir = real_config.getdefault('pants_supportdir')
link_tree(os.path.relpath(os.path.join(support_dir, 'ivy'), self.real_build_root), force=True)
def execute(self):
registered_tools = JvmToolMixin.get_registered_tools()
if registered_tools:
# Map of scope -> (map of key -> callback).
callback_product_map = self.context.products.get_data('jvm_build_tools_classpath_callbacks',
init_func=lambda: defaultdict(dict))
# We leave a callback in the products map because we want these Ivy calls
# to be done lazily (they might never actually get executed) and we want
# to hit Task.invalidated (called in Task._ivy_resolve) on the instance of
# BootstrapJvmTools rather than the instance of whatever class requires
# the bootstrap tools. It would be awkward and possibly incorrect to call
# self.invalidated twice on a Task that does meaningful invalidation on its
# targets. -pl
for jvm_tool in registered_tools:
dep_spec = jvm_tool.dep_spec(self.context.options)
callback = self.cached_bootstrap_classpath_callback(dep_spec, jvm_tool)
callback_product_map[jvm_tool.scope][jvm_tool.key] = callback
if self.get_options().eager:
with self.context.new_workunit('eager'):
for scope, callbacks_by_key in callback_product_map.items():
for key, callback in callbacks_by_key.items():
try:
callback()
except self.ToolUnderspecified:
pass # We don't want to fail for placeholder registrations
def execute(self):
registered_tools = JvmToolMixin.get_registered_tools()
if registered_tools:
# Map of scope -> (map of key -> callback).
callback_product_map = self.context.products.get_data(
"jvm_build_tools_classpath_callbacks",
init_func=lambda: defaultdict(dict))
# We leave a callback in the products map because we want these Ivy calls
# to be done lazily (they might never actually get executed) and we want
# to hit Task.invalidated (called in Task._ivy_resolve) on the instance of
# BootstrapJvmTools rather than the instance of whatever class requires
# the bootstrap tools. It would be awkward and possibly incorrect to call
# self.invalidated twice on a Task that does meaningful invalidation on its
# targets. -pl
for jvm_tool in registered_tools:
dep_spec = jvm_tool.dep_spec(self.context.options)
callback = self.cached_bootstrap_classpath_callback(
dep_spec, jvm_tool)
callback_product_map[jvm_tool.scope][jvm_tool.key] = callback
if self.get_options().eager:
with self.context.new_workunit("eager"):
for scope, callbacks_by_key in callback_product_map.items(
):
for key, callback in callbacks_by_key.items():
try:
callback()
except self.ToolUnderspecified:
pass # We don't want to fail for placeholder registrations
def setUp(self):
"""
:API: public
"""
super().setUp()
# Use a synthetic subclass for proper isolation when bootstrapping within the test.
bootstrap_scope = "bootstrap_scope"
self.bootstrap_task_type = self.synthesize_task_subtype(
BootstrapJvmTools, bootstrap_scope)
JvmToolMixin.reset_registered_tools()
# Set some options:
# 1. Cap BootstrapJvmTools memory usage in tests. The Xmx was empirically arrived upon using
# -Xloggc and verifying no full gcs for a test using the full gamut of resolving a multi-jar
# tool, constructing a fat jar and then shading that fat jar.
#
# 2. Allow tests to read/write tool jars from the real artifact cache, so they don't
# each have to resolve and shade them every single time, which is a huge slowdown.
# Note that local artifact cache writes are atomic, so it's fine for multiple concurrent
# tests to write to it.
#
# Note that we don't have access to the invoking pants instance's options, so we assume that
# its artifact cache is in the standard location. If it isn't, worst case the tests will
# populate a second cache at the standard location, which is no big deal.
# TODO: We really need a straightforward way for pants's own tests to get to the enclosing
# pants instance's options values.
artifact_caches = [
os.path.join(get_pants_cachedir(), "artifact_cache")
]
self.set_options_for_scope(
bootstrap_scope,
execution_strategy=NailgunTask.ExecutionStrategy.subprocess,
jvm_options=["-Xmx128m"],
)
self.set_options_for_scope(f"cache.{bootstrap_scope}",
read_from=artifact_caches,
write_to=artifact_caches)
# Copy into synthetic build-root
shutil.copy("BUILD.tools", self.build_root)
build_root_third_party = os.path.join(self.build_root, "3rdparty")
safe_mkdir(build_root_third_party)
shutil.copy(os.path.join("3rdparty", "BUILD"), build_root_third_party)
Bootstrapper.reset_instance()
def get_alternate_target_roots(cls, options, address_mapper, build_graph):
processed = set()
for jvm_tool in JvmToolMixin.get_registered_tools():
dep_spec = jvm_tool.dep_spec(options)
dep_address = Address.parse(dep_spec)
# Some JVM tools are requested multiple times, we only need to handle them once.
if dep_address not in processed:
processed.add(dep_address)
try:
if build_graph.resolve_address(dep_address):
# The user has defined a tool classpath override - we let that stand.
continue
except AddressLookupError as e:
if jvm_tool.classpath is None:
raise cls._tool_resolve_error(e, dep_spec, jvm_tool)
else:
if not jvm_tool.is_default(options):
# The user specified a target spec for this jvm tool that doesn't actually exist.
# We want to error out here instead of just silently using the default option while
# appearing to respect their config.
raise cls.ToolResolveError(dedent("""
Failed to resolve target for tool: {tool}. This target was obtained from
option {option} in scope {scope}.
Make sure you didn't make a typo in the tool's address. You specified that the
tool should use the target found at "{tool}".
This target has a default classpath configured, so you can simply remove:
[{scope}]
{option}: {tool}
from pants.ini (or any other config file) to use the default tool.
The default classpath is: {default_classpath}
Note that tool target addresses in pants.ini should be specified *without* quotes.
""").strip().format(tool=dep_spec,
option=jvm_tool.key,
scope=jvm_tool.scope,
default_classpath=':'.join(map(str, jvm_tool.classpath or ()))))
if jvm_tool.classpath:
tool_classpath_target = JarLibrary(name=dep_address.target_name,
address=dep_address,
build_graph=build_graph,
jars=jvm_tool.classpath)
else:
# The tool classpath is empty by default, so we just inject a dummy target that
# ivy resolves as the empty list classpath. JarLibrary won't do since it requires
# one or more jars, so we just pick a target type ivy has no resolve work to do for.
tool_classpath_target = Target(name=dep_address.target_name,
address=dep_address,
build_graph=build_graph)
build_graph.inject_target(tool_classpath_target, synthetic=True)
# We use the trick of not returning alternate roots, but instead just filling the dep_spec
# holes with a JarLibrary built from a tool's default classpath JarDependency list if there is
# no over-riding targets present. This means we do modify the build_graph, but we at least do
# it at a time in the engine lifecycle cut out for handling that.
return None
def _alternate_target_roots(cls, options, address_mapper, build_graph):
processed = set()
for jvm_tool in JvmToolMixin.get_registered_tools():
dep_spec = jvm_tool.dep_spec(options)
dep_address = Address.parse(dep_spec)
# Some JVM tools are requested multiple times, we only need to handle them once.
if dep_address not in processed:
processed.add(dep_address)
try:
if build_graph.resolve_address(dep_address):
# The user has defined a tool classpath override - we let that stand.
continue
except AddressLookupError as e:
if jvm_tool.classpath is None:
raise cls._tool_resolve_error(e, dep_spec, jvm_tool)
else:
if not jvm_tool.is_default(options):
# The user specified a target spec for this jvm tool that doesn't actually exist.
# We want to error out here instead of just silently using the default option while
# appearing to respect their config.
raise cls.ToolResolveError(dedent("""
Failed to resolve target for tool: {tool}. This target was obtained from
option {option} in scope {scope}.
Make sure you didn't make a typo in the tool's address. You specified that the
tool should use the target found at "{tool}".
This target has a default classpath configured, so you can simply remove:
[{scope}]
{option}: {tool}
from pants.ini (or any other config file) to use the default tool.
The default classpath is: {default_classpath}
Note that tool target addresses in pants.ini should be specified *without* quotes.
""").strip().format(tool=dep_spec,
option=jvm_tool.key,
scope=jvm_tool.scope,
default_classpath=':'.join(map(str, jvm_tool.classpath or ()))))
if jvm_tool.classpath:
tool_classpath_target = JarLibrary(name=dep_address.target_name,
address=dep_address,
build_graph=build_graph,
jars=jvm_tool.classpath)
else:
# The tool classpath is empty by default, so we just inject a dummy target that
# ivy resolves as the empty list classpath. JarLibrary won't do since it requires
# one or more jars, so we just pick a target type ivy has no resolve work to do for.
tool_classpath_target = Target(name=dep_address.target_name,
address=dep_address,
build_graph=build_graph)
build_graph.inject_target(tool_classpath_target, synthetic=True)
# We use the trick of not returning alternate roots, but instead just filling the dep_spec
# holes with a JarLibrary built from a tool's default classpath JarDependency list if there is
# no over-riding targets present. This means we do modify the build_graph, but we at least do
# it at a time in the engine lifecycle cut out for handling that.
return None
def setUp(self):
super(JvmToolTaskTestBase, self).setUp()
# Use a synthetic subclass for proper isolation when bootstrapping within the test.
bootstrap_scope = 'bootstrap_scope'
self.bootstrap_task_type = self.synthesize_task_subtype(BootstrapJvmTools, bootstrap_scope)
JvmToolMixin.reset_registered_tools()
# Cap BootstrapJvmTools memory usage in tests. The Xmx was empirically arrived upon using
# -Xloggc and verifying no full gcs for a test using the full gamut of resolving a multi-jar
# tool, constructing a fat jar and then shading that fat jar.
self.set_options_for_scope(bootstrap_scope, jvm_options=['-Xmx128m'])
# Tool option defaults currently point to targets in the real BUILD.tools, so we copy it
# into our test workspace.
shutil.copy(os.path.join(self.real_build_root, 'BUILD.tools'), self.build_root)
Bootstrapper.reset_instance()
def execute(self):
context = self.context
if JvmToolMixin.get_registered_tools():
# Map of scope -> (map of key -> callback).
callback_product_map = (context.products.get_data('jvm_build_tools_classpath_callbacks') or
defaultdict(dict))
# We leave a callback in the products map because we want these Ivy calls
# to be done lazily (they might never actually get executed) and we want
# to hit Task.invalidated (called in Task.ivy_resolve) on the instance of
# BootstrapJvmTools rather than the instance of whatever class requires
# the bootstrap tools. It would be awkward and possibly incorrect to call
# self.invalidated twice on a Task that does meaningful invalidation on its
# targets. -pl
for scope, key, main, custom_rules in JvmToolMixin.get_registered_tools():
option = key.replace('-', '_')
deplist = self.context.options.for_scope(scope)[option]
callback_product_map[scope][key] = self.cached_bootstrap_classpath_callback(
key, scope, deplist, main=main, custom_rules=custom_rules)
context.products.safe_create_data('jvm_build_tools_classpath_callbacks',
lambda: callback_product_map)
def execute(self):
registered_tools = JvmToolMixin.get_registered_tools()
if registered_tools:
# Map of scope -> (map of key -> callback).
callback_product_map = self.context.products.get_data('jvm_build_tools_classpath_callbacks',
init_func=lambda: defaultdict(dict))
# We leave a callback in the products map because we want these Ivy calls
# to be done lazily (they might never actually get executed) and we want
# to hit Task.invalidated (called in Task._ivy_resolve) on the instance of
# BootstrapJvmTools rather than the instance of whatever class requires
# the bootstrap tools. It would be awkward and possibly incorrect to call
# self.invalidated twice on a Task that does meaningful invalidation on its
# targets. -pl
for jvm_tool in registered_tools:
dep_spec = jvm_tool.dep_spec(self.context.options)
callback = self.cached_bootstrap_classpath_callback(dep_spec, jvm_tool)
callback_product_map[jvm_tool.scope][jvm_tool.key] = callback
def execute(self):
registered_tools = JvmToolMixin.get_registered_tools()
if registered_tools:
# Map of scope -> (map of key -> callback).
callback_product_map = self.context.products.get_data(
"jvm_build_tools_classpath_callbacks", init_func=lambda: defaultdict(dict)
)
# We leave a callback in the products map because we want these Ivy calls
# to be done lazily (they might never actually get executed) and we want
# to hit Task.invalidated (called in Task.ivy_resolve) on the instance of
# BootstrapJvmTools rather than the instance of whatever class requires
# the bootstrap tools. It would be awkward and possibly incorrect to call
# self.invalidated twice on a Task that does meaningful invalidation on its
# targets. -pl
for jvm_tool in registered_tools:
dep_spec = jvm_tool.dep_spec(self.context.options)
callback = self.cached_bootstrap_classpath_callback(dep_spec, jvm_tool)
callback_product_map[jvm_tool.scope][jvm_tool.key] = callback
def execute(self):
registered_tools = JvmToolMixin.get_registered_tools()
if registered_tools:
# Map of scope -> (map of key -> callback).
callback_product_map = self.context.products.get_data(
"jvm_build_tools_classpath_callbacks",
init_func=lambda: defaultdict(dict))
# We leave a callback in the products map because we want these resolver calls
# to be done lazily (they might never actually get executed).
for jvm_tool in registered_tools:
dep_spec = jvm_tool.dep_spec(self.context.options)
callback = self.cached_bootstrap_classpath_callback(
dep_spec, jvm_tool)
callback_product_map[jvm_tool.scope][jvm_tool.key] = callback
if self.get_options().eager:
with self.context.new_workunit("eager"):
for scope, callbacks_by_key in callback_product_map.items(
):
for key, callback in callbacks_by_key.items():
try:
callback()
except self.ToolUnderspecified:
pass # We don't want to fail for placeholder registrations
def _alternate_target_roots(cls, options, address_mapper, build_graph):
processed = set()
for jvm_tool in JvmToolMixin.get_registered_tools():
dep_spec = jvm_tool.dep_spec(options)
dep_address = Address.parse(dep_spec)
# Some JVM tools are requested multiple times, we only need to handle them once.
if dep_address not in processed:
processed.add(dep_address)
try:
if build_graph.contains_address(dep_address) or address_mapper.resolve(dep_address):
# The user has defined a tool classpath override - we let that stand.
continue
except AddressLookupError as e:
if jvm_tool.classpath is None:
raise cls._tool_resolve_error(e, dep_spec, jvm_tool)
else:
if jvm_tool.classpath:
tool_classpath_target = JarLibrary(
name=dep_address.target_name,
address=dep_address,
build_graph=build_graph,
jars=jvm_tool.classpath,
)
else:
# The tool classpath is empty by default, so we just inject a dummy target that
# ivy resolves as the empty list classpath. JarLibrary won't do since it requires
# one or more jars, so we just pick a target type ivy has no resolve work to do for.
tool_classpath_target = Target(
name=dep_address.target_name, address=dep_address, build_graph=build_graph
)
build_graph.inject_target(tool_classpath_target)
# We use the trick of not returning alternate roots, but instead just filling the dep_spec
# holes with a JarLibrary built from a tool's default classpath JarDependency list if there is
# no over-riding targets present. This means we do modify the build_graph, but we at least do
# it at a time in the engine lifecycle cut out for handling that.
return None