def setUp(self): super(JvmToolTaskTestBase, self).setUp() # Use a synthetic subclass for proper isolation when bootstrapping within the test. bootstrap_scope = 'bootstrap_scope' self.bootstrap_task_type = self.synthesize_task_subtype(BootstrapJvmTools, bootstrap_scope) JvmToolMixin.reset_registered_tools() # Set some options: # 1. Cap BootstrapJvmTools memory usage in tests. The Xmx was empirically arrived upon using # -Xloggc and verifying no full gcs for a test using the full gamut of resolving a multi-jar # tool, constructing a fat jar and then shading that fat jar. # # 2. Allow tests to read/write tool jars from the real artifact cache, so they don't # each have to resolve and shade them every single time, which is a huge slowdown. # Note that local artifact cache writes are atomic, so it's fine for multiple concurrent # tests to write to it. # # Note that we don't have access to the invoking pants instance's options, so we assume that # its artifact cache is in the standard location. If it isn't, worst case the tests will # populate a second cache at the standard location, which is no big deal. # TODO: We really need a straightforward way for pants's own tests to get to the enclosing # pants instance's options values. artifact_caches = [os.path.join(get_pants_cachedir(), 'artifact_cache')] self.set_options_for_scope(bootstrap_scope, jvm_options=['-Xmx128m']) self.set_options_for_scope('cache.{}'.format(bootstrap_scope), read_from=artifact_caches, write_to=artifact_caches) # Tool option defaults currently point to targets in the real BUILD.tools, so we copy it # into our test workspace. shutil.copy(os.path.join(self.real_build_root, 'BUILD.tools'), self.build_root) Bootstrapper.reset_instance()
def execute(self): context = self.context if JvmToolMixin.get_registered_tools(): # Map of scope -> (map of key -> callback). callback_product_map = (context.products.get_data( 'jvm_build_tools_classpath_callbacks') or defaultdict(dict)) # We leave a callback in the products map because we want these Ivy calls # to be done lazily (they might never actually get executed) and we want # to hit Task.invalidated (called in Task.ivy_resolve) on the instance of # BootstrapJvmTools rather than the instance of whatever class requires # the bootstrap tools. It would be awkward and possibly incorrect to call # self.invalidated twice on a Task that does meaningful invalidation on its # targets. -pl for scope, key, main, custom_rules in JvmToolMixin.get_registered_tools( ): option = key.replace('-', '_') deplist = self.context.options.for_scope(scope)[option] callback_product_map[scope][ key] = self.cached_bootstrap_classpath_callback( key, scope, deplist, main=main, custom_rules=custom_rules) context.products.safe_create_data( 'jvm_build_tools_classpath_callbacks', lambda: callback_product_map)
def setUp(self): # Ensure we get a read of the real pants.ini config Config.reset_default_bootstrap_option_values() real_config = Config.from_cache() super(JvmToolTaskTestBase, self).setUp() # Use a synthetic subclass for bootstrapping within the test, to isolate this from # any bootstrapping the pants run executing the test might need. self.bootstrap_task_type, bootstrap_scope = self.synthesize_task_subtype(BootstrapJvmTools) JvmToolMixin.reset_registered_tools() # Cap BootstrapJvmTools memory usage in tests. The Xmx was empirically arrived upon using # -Xloggc and verifying no full gcs for a test using the full gamut of resolving a multi-jar # tool, constructing a fat jar and then shading that fat jar. self.set_options_for_scope(bootstrap_scope, jvm_options=['-Xmx128m']) def link_or_copy(src, dest): try: os.link(src, dest) except OSError as e: if e.errno == errno.EXDEV: shutil.copy(src, dest) else: raise e def link(path, optional=False, force=False): src = os.path.join(self.real_build_root, path) if not optional or os.path.exists(src): dest = os.path.join(self.build_root, path) safe_mkdir(os.path.dirname(dest)) try: link_or_copy(src, dest) except OSError as e: if force and e.errno == errno.EEXIST: os.unlink(dest) link_or_copy(src, dest) else: raise e return dest def link_tree(path, optional=False, force=False): src = os.path.join(self.real_build_root, path) if not optional or os.path.exists(src): for abspath, dirs, files in safe_walk(src): for f in files: link(os.path.relpath(os.path.join(abspath, f), self.real_build_root), force=force) # TODO(John Sirois): Find a way to do this cleanly link('pants.ini', force=True) # TODO(pl): Note that this pulls in a big chunk of the hairball to every test that # depends on it, because BUILD contains source_roots that specify a variety of types # from different backends. link('BUILD', force=True) link('BUILD.tools', force=True) support_dir = real_config.getdefault('pants_supportdir') link_tree(os.path.relpath(os.path.join(support_dir, 'ivy'), self.real_build_root), force=True)
def execute(self): registered_tools = JvmToolMixin.get_registered_tools() if registered_tools: # Map of scope -> (map of key -> callback). callback_product_map = self.context.products.get_data('jvm_build_tools_classpath_callbacks', init_func=lambda: defaultdict(dict)) # We leave a callback in the products map because we want these Ivy calls # to be done lazily (they might never actually get executed) and we want # to hit Task.invalidated (called in Task._ivy_resolve) on the instance of # BootstrapJvmTools rather than the instance of whatever class requires # the bootstrap tools. It would be awkward and possibly incorrect to call # self.invalidated twice on a Task that does meaningful invalidation on its # targets. -pl for jvm_tool in registered_tools: dep_spec = jvm_tool.dep_spec(self.context.options) callback = self.cached_bootstrap_classpath_callback(dep_spec, jvm_tool) callback_product_map[jvm_tool.scope][jvm_tool.key] = callback if self.get_options().eager: with self.context.new_workunit('eager'): for scope, callbacks_by_key in callback_product_map.items(): for key, callback in callbacks_by_key.items(): try: callback() except self.ToolUnderspecified: pass # We don't want to fail for placeholder registrations
def execute(self): registered_tools = JvmToolMixin.get_registered_tools() if registered_tools: # Map of scope -> (map of key -> callback). callback_product_map = self.context.products.get_data( "jvm_build_tools_classpath_callbacks", init_func=lambda: defaultdict(dict)) # We leave a callback in the products map because we want these Ivy calls # to be done lazily (they might never actually get executed) and we want # to hit Task.invalidated (called in Task._ivy_resolve) on the instance of # BootstrapJvmTools rather than the instance of whatever class requires # the bootstrap tools. It would be awkward and possibly incorrect to call # self.invalidated twice on a Task that does meaningful invalidation on its # targets. -pl for jvm_tool in registered_tools: dep_spec = jvm_tool.dep_spec(self.context.options) callback = self.cached_bootstrap_classpath_callback( dep_spec, jvm_tool) callback_product_map[jvm_tool.scope][jvm_tool.key] = callback if self.get_options().eager: with self.context.new_workunit("eager"): for scope, callbacks_by_key in callback_product_map.items( ): for key, callback in callbacks_by_key.items(): try: callback() except self.ToolUnderspecified: pass # We don't want to fail for placeholder registrations
def setUp(self): """ :API: public """ super().setUp() # Use a synthetic subclass for proper isolation when bootstrapping within the test. bootstrap_scope = "bootstrap_scope" self.bootstrap_task_type = self.synthesize_task_subtype( BootstrapJvmTools, bootstrap_scope) JvmToolMixin.reset_registered_tools() # Set some options: # 1. Cap BootstrapJvmTools memory usage in tests. The Xmx was empirically arrived upon using # -Xloggc and verifying no full gcs for a test using the full gamut of resolving a multi-jar # tool, constructing a fat jar and then shading that fat jar. # # 2. Allow tests to read/write tool jars from the real artifact cache, so they don't # each have to resolve and shade them every single time, which is a huge slowdown. # Note that local artifact cache writes are atomic, so it's fine for multiple concurrent # tests to write to it. # # Note that we don't have access to the invoking pants instance's options, so we assume that # its artifact cache is in the standard location. If it isn't, worst case the tests will # populate a second cache at the standard location, which is no big deal. # TODO: We really need a straightforward way for pants's own tests to get to the enclosing # pants instance's options values. artifact_caches = [ os.path.join(get_pants_cachedir(), "artifact_cache") ] self.set_options_for_scope( bootstrap_scope, execution_strategy=NailgunTask.ExecutionStrategy.subprocess, jvm_options=["-Xmx128m"], ) self.set_options_for_scope(f"cache.{bootstrap_scope}", read_from=artifact_caches, write_to=artifact_caches) # Copy into synthetic build-root shutil.copy("BUILD.tools", self.build_root) build_root_third_party = os.path.join(self.build_root, "3rdparty") safe_mkdir(build_root_third_party) shutil.copy(os.path.join("3rdparty", "BUILD"), build_root_third_party) Bootstrapper.reset_instance()
def get_alternate_target_roots(cls, options, address_mapper, build_graph): processed = set() for jvm_tool in JvmToolMixin.get_registered_tools(): dep_spec = jvm_tool.dep_spec(options) dep_address = Address.parse(dep_spec) # Some JVM tools are requested multiple times, we only need to handle them once. if dep_address not in processed: processed.add(dep_address) try: if build_graph.resolve_address(dep_address): # The user has defined a tool classpath override - we let that stand. continue except AddressLookupError as e: if jvm_tool.classpath is None: raise cls._tool_resolve_error(e, dep_spec, jvm_tool) else: if not jvm_tool.is_default(options): # The user specified a target spec for this jvm tool that doesn't actually exist. # We want to error out here instead of just silently using the default option while # appearing to respect their config. raise cls.ToolResolveError(dedent(""" Failed to resolve target for tool: {tool}. This target was obtained from option {option} in scope {scope}. Make sure you didn't make a typo in the tool's address. You specified that the tool should use the target found at "{tool}". This target has a default classpath configured, so you can simply remove: [{scope}] {option}: {tool} from pants.ini (or any other config file) to use the default tool. The default classpath is: {default_classpath} Note that tool target addresses in pants.ini should be specified *without* quotes. """).strip().format(tool=dep_spec, option=jvm_tool.key, scope=jvm_tool.scope, default_classpath=':'.join(map(str, jvm_tool.classpath or ())))) if jvm_tool.classpath: tool_classpath_target = JarLibrary(name=dep_address.target_name, address=dep_address, build_graph=build_graph, jars=jvm_tool.classpath) else: # The tool classpath is empty by default, so we just inject a dummy target that # ivy resolves as the empty list classpath. JarLibrary won't do since it requires # one or more jars, so we just pick a target type ivy has no resolve work to do for. tool_classpath_target = Target(name=dep_address.target_name, address=dep_address, build_graph=build_graph) build_graph.inject_target(tool_classpath_target, synthetic=True) # We use the trick of not returning alternate roots, but instead just filling the dep_spec # holes with a JarLibrary built from a tool's default classpath JarDependency list if there is # no over-riding targets present. This means we do modify the build_graph, but we at least do # it at a time in the engine lifecycle cut out for handling that. return None
def _alternate_target_roots(cls, options, address_mapper, build_graph): processed = set() for jvm_tool in JvmToolMixin.get_registered_tools(): dep_spec = jvm_tool.dep_spec(options) dep_address = Address.parse(dep_spec) # Some JVM tools are requested multiple times, we only need to handle them once. if dep_address not in processed: processed.add(dep_address) try: if build_graph.resolve_address(dep_address): # The user has defined a tool classpath override - we let that stand. continue except AddressLookupError as e: if jvm_tool.classpath is None: raise cls._tool_resolve_error(e, dep_spec, jvm_tool) else: if not jvm_tool.is_default(options): # The user specified a target spec for this jvm tool that doesn't actually exist. # We want to error out here instead of just silently using the default option while # appearing to respect their config. raise cls.ToolResolveError(dedent(""" Failed to resolve target for tool: {tool}. This target was obtained from option {option} in scope {scope}. Make sure you didn't make a typo in the tool's address. You specified that the tool should use the target found at "{tool}". This target has a default classpath configured, so you can simply remove: [{scope}] {option}: {tool} from pants.ini (or any other config file) to use the default tool. The default classpath is: {default_classpath} Note that tool target addresses in pants.ini should be specified *without* quotes. """).strip().format(tool=dep_spec, option=jvm_tool.key, scope=jvm_tool.scope, default_classpath=':'.join(map(str, jvm_tool.classpath or ())))) if jvm_tool.classpath: tool_classpath_target = JarLibrary(name=dep_address.target_name, address=dep_address, build_graph=build_graph, jars=jvm_tool.classpath) else: # The tool classpath is empty by default, so we just inject a dummy target that # ivy resolves as the empty list classpath. JarLibrary won't do since it requires # one or more jars, so we just pick a target type ivy has no resolve work to do for. tool_classpath_target = Target(name=dep_address.target_name, address=dep_address, build_graph=build_graph) build_graph.inject_target(tool_classpath_target, synthetic=True) # We use the trick of not returning alternate roots, but instead just filling the dep_spec # holes with a JarLibrary built from a tool's default classpath JarDependency list if there is # no over-riding targets present. This means we do modify the build_graph, but we at least do # it at a time in the engine lifecycle cut out for handling that. return None
def setUp(self): super(JvmToolTaskTestBase, self).setUp() # Use a synthetic subclass for proper isolation when bootstrapping within the test. bootstrap_scope = 'bootstrap_scope' self.bootstrap_task_type = self.synthesize_task_subtype(BootstrapJvmTools, bootstrap_scope) JvmToolMixin.reset_registered_tools() # Cap BootstrapJvmTools memory usage in tests. The Xmx was empirically arrived upon using # -Xloggc and verifying no full gcs for a test using the full gamut of resolving a multi-jar # tool, constructing a fat jar and then shading that fat jar. self.set_options_for_scope(bootstrap_scope, jvm_options=['-Xmx128m']) # Tool option defaults currently point to targets in the real BUILD.tools, so we copy it # into our test workspace. shutil.copy(os.path.join(self.real_build_root, 'BUILD.tools'), self.build_root) Bootstrapper.reset_instance()
def execute(self): context = self.context if JvmToolMixin.get_registered_tools(): # Map of scope -> (map of key -> callback). callback_product_map = (context.products.get_data('jvm_build_tools_classpath_callbacks') or defaultdict(dict)) # We leave a callback in the products map because we want these Ivy calls # to be done lazily (they might never actually get executed) and we want # to hit Task.invalidated (called in Task.ivy_resolve) on the instance of # BootstrapJvmTools rather than the instance of whatever class requires # the bootstrap tools. It would be awkward and possibly incorrect to call # self.invalidated twice on a Task that does meaningful invalidation on its # targets. -pl for scope, key, main, custom_rules in JvmToolMixin.get_registered_tools(): option = key.replace('-', '_') deplist = self.context.options.for_scope(scope)[option] callback_product_map[scope][key] = self.cached_bootstrap_classpath_callback( key, scope, deplist, main=main, custom_rules=custom_rules) context.products.safe_create_data('jvm_build_tools_classpath_callbacks', lambda: callback_product_map)
def execute(self): registered_tools = JvmToolMixin.get_registered_tools() if registered_tools: # Map of scope -> (map of key -> callback). callback_product_map = self.context.products.get_data('jvm_build_tools_classpath_callbacks', init_func=lambda: defaultdict(dict)) # We leave a callback in the products map because we want these Ivy calls # to be done lazily (they might never actually get executed) and we want # to hit Task.invalidated (called in Task._ivy_resolve) on the instance of # BootstrapJvmTools rather than the instance of whatever class requires # the bootstrap tools. It would be awkward and possibly incorrect to call # self.invalidated twice on a Task that does meaningful invalidation on its # targets. -pl for jvm_tool in registered_tools: dep_spec = jvm_tool.dep_spec(self.context.options) callback = self.cached_bootstrap_classpath_callback(dep_spec, jvm_tool) callback_product_map[jvm_tool.scope][jvm_tool.key] = callback
def execute(self): registered_tools = JvmToolMixin.get_registered_tools() if registered_tools: # Map of scope -> (map of key -> callback). callback_product_map = self.context.products.get_data( "jvm_build_tools_classpath_callbacks", init_func=lambda: defaultdict(dict) ) # We leave a callback in the products map because we want these Ivy calls # to be done lazily (they might never actually get executed) and we want # to hit Task.invalidated (called in Task.ivy_resolve) on the instance of # BootstrapJvmTools rather than the instance of whatever class requires # the bootstrap tools. It would be awkward and possibly incorrect to call # self.invalidated twice on a Task that does meaningful invalidation on its # targets. -pl for jvm_tool in registered_tools: dep_spec = jvm_tool.dep_spec(self.context.options) callback = self.cached_bootstrap_classpath_callback(dep_spec, jvm_tool) callback_product_map[jvm_tool.scope][jvm_tool.key] = callback
def execute(self): registered_tools = JvmToolMixin.get_registered_tools() if registered_tools: # Map of scope -> (map of key -> callback). callback_product_map = self.context.products.get_data( "jvm_build_tools_classpath_callbacks", init_func=lambda: defaultdict(dict)) # We leave a callback in the products map because we want these resolver calls # to be done lazily (they might never actually get executed). for jvm_tool in registered_tools: dep_spec = jvm_tool.dep_spec(self.context.options) callback = self.cached_bootstrap_classpath_callback( dep_spec, jvm_tool) callback_product_map[jvm_tool.scope][jvm_tool.key] = callback if self.get_options().eager: with self.context.new_workunit("eager"): for scope, callbacks_by_key in callback_product_map.items( ): for key, callback in callbacks_by_key.items(): try: callback() except self.ToolUnderspecified: pass # We don't want to fail for placeholder registrations
def _alternate_target_roots(cls, options, address_mapper, build_graph): processed = set() for jvm_tool in JvmToolMixin.get_registered_tools(): dep_spec = jvm_tool.dep_spec(options) dep_address = Address.parse(dep_spec) # Some JVM tools are requested multiple times, we only need to handle them once. if dep_address not in processed: processed.add(dep_address) try: if build_graph.contains_address(dep_address) or address_mapper.resolve(dep_address): # The user has defined a tool classpath override - we let that stand. continue except AddressLookupError as e: if jvm_tool.classpath is None: raise cls._tool_resolve_error(e, dep_spec, jvm_tool) else: if jvm_tool.classpath: tool_classpath_target = JarLibrary( name=dep_address.target_name, address=dep_address, build_graph=build_graph, jars=jvm_tool.classpath, ) else: # The tool classpath is empty by default, so we just inject a dummy target that # ivy resolves as the empty list classpath. JarLibrary won't do since it requires # one or more jars, so we just pick a target type ivy has no resolve work to do for. tool_classpath_target = Target( name=dep_address.target_name, address=dep_address, build_graph=build_graph ) build_graph.inject_target(tool_classpath_target) # We use the trick of not returning alternate roots, but instead just filling the dep_spec # holes with a JarLibrary built from a tool's default classpath JarDependency list if there is # no over-riding targets present. This means we do modify the build_graph, but we at least do # it at a time in the engine lifecycle cut out for handling that. return None