def setUp(self): super(JvmToolTaskTestBase, self).setUp() # Use a synthetic subclass for proper isolation when bootstrapping within the test. bootstrap_scope = 'bootstrap_scope' self.bootstrap_task_type = self.synthesize_task_subtype(BootstrapJvmTools, bootstrap_scope) JvmToolMixin.reset_registered_tools() # Set some options: # 1. Cap BootstrapJvmTools memory usage in tests. The Xmx was empirically arrived upon using # -Xloggc and verifying no full gcs for a test using the full gamut of resolving a multi-jar # tool, constructing a fat jar and then shading that fat jar. # # 2. Allow tests to read/write tool jars from the real artifact cache, so they don't # each have to resolve and shade them every single time, which is a huge slowdown. # Note that local artifact cache writes are atomic, so it's fine for multiple concurrent # tests to write to it. # # Note that we don't have access to the invoking pants instance's options, so we assume that # its artifact cache is in the standard location. If it isn't, worst case the tests will # populate a second cache at the standard location, which is no big deal. # TODO: We really need a straightforward way for pants's own tests to get to the enclosing # pants instance's options values. artifact_caches = [os.path.join(get_pants_cachedir(), 'artifact_cache')] self.set_options_for_scope(bootstrap_scope, jvm_options=['-Xmx128m']) self.set_options_for_scope('cache.{}'.format(bootstrap_scope), read_from=artifact_caches, write_to=artifact_caches) # Tool option defaults currently point to targets in the real BUILD.tools, so we copy it # into our test workspace. shutil.copy(os.path.join(self.real_build_root, 'BUILD.tools'), self.build_root) Bootstrapper.reset_instance()
def execute_junit_runner(self, content): # Create the temporary base test directory test_rel_path = 'tests/java/org/pantsbuild/foo' test_abs_path = os.path.join(self.build_root, test_rel_path) self.create_dir(test_rel_path) # Generate the temporary java test source code. test_java_file_rel_path = os.path.join(test_rel_path, 'FooTest.java') test_java_file_abs_path = os.path.join(self.build_root, test_java_file_rel_path) self.create_file(test_java_file_rel_path, content) # Invoke ivy to resolve classpath for junit. distribution = Distribution.cached(jdk=True) executor = SubprocessExecutor(distribution=distribution) classpath_file_abs_path = os.path.join(test_abs_path, 'junit.classpath') with subsystem_instance(IvySubsystem) as ivy_subsystem: ivy = Bootstrapper(ivy_subsystem=ivy_subsystem).ivy() ivy.execute(args=['-cachepath', classpath_file_abs_path, '-dependency', 'junit', 'junit-dep', '4.10'], executor=executor) with open(classpath_file_abs_path) as fp: classpath = fp.read() # Now directly invoking javac to compile the test java code into java class # so later we can inject the class into products mapping for JUnitRun to execute # the test on. javac = distribution.binary('javac') subprocess.check_call( [javac, '-d', test_abs_path, '-cp', classpath, test_java_file_abs_path]) # Create a java_tests target and a synthetic resource target. java_tests = self.create_library(test_rel_path, 'java_tests', 'foo_test', ['FooTest.java']) resources = self.make_target('some_resources', Resources) # Set the context with the two targets, one java_tests target and # one synthetic resources target. # The synthetic resources target is to make sure we won't regress # in the future with bug like https://github.com/pantsbuild/pants/issues/508. Note # in that bug, the resources target must be the first one in the list. context = self.context(target_roots=[resources, java_tests]) # Before we run the task, we need to inject the "classes_by_target" with # the compiled test java classes that JUnitRun will know which test # classes to execute. In a normal run, this "classes_by_target" will be # populated by java compiling step. class_products = context.products.get_data( 'classes_by_target', lambda: defaultdict(MultipleRootedProducts)) java_tests_products = MultipleRootedProducts() java_tests_products.add_rel_paths(test_abs_path, ['FooTest.class']) class_products[java_tests] = java_tests_products # Also we need to add the FooTest.class's classpath to the compile_classpath # products data mapping so JUnitRun will be able to add that into the final # classpath under which the junit will be executed. self.populate_compile_classpath(context=context, classpath=[test_abs_path]) # Finally execute the task. self.execute(context)
def execute_junit_runner(self, content): # Create the temporary base test directory test_rel_path = 'tests/java/org/pantsbuild/foo' test_abs_path = os.path.join(self.build_root, test_rel_path) self.create_dir(test_rel_path) # Generate the temporary java test source code. test_java_file_rel_path = os.path.join(test_rel_path, 'FooTest.java') test_java_file_abs_path = os.path.join(self.build_root, test_java_file_rel_path) self.create_file(test_java_file_rel_path, content) # Invoke ivy to resolve classpath for junit. classpath_file_abs_path = os.path.join(test_abs_path, 'junit.classpath') with subsystem_instance(IvySubsystem) as ivy_subsystem: distribution = DistributionLocator.cached(jdk=True) ivy = Bootstrapper(ivy_subsystem=ivy_subsystem).ivy() ivy.execute(args=['-cachepath', classpath_file_abs_path, '-dependency', 'junit', 'junit-dep', '4.10'], executor=SubprocessExecutor(distribution=distribution)) with open(classpath_file_abs_path) as fp: classpath = fp.read() # Now directly invoking javac to compile the test java code into java class # so later we can inject the class into products mapping for JUnitRun to execute # the test on. javac = distribution.binary('javac') subprocess.check_call( [javac, '-d', test_abs_path, '-cp', classpath, test_java_file_abs_path]) # Create a java_tests target and a synthetic resource target. java_tests = self.create_library(test_rel_path, 'java_tests', 'foo_test', ['FooTest.java']) resources = self.make_target('some_resources', Resources) # Set the context with the two targets, one java_tests target and # one synthetic resources target. # The synthetic resources target is to make sure we won't regress # in the future with bug like https://github.com/pantsbuild/pants/issues/508. Note # in that bug, the resources target must be the first one in the list. context = self.context(target_roots=[resources, java_tests]) # Before we run the task, we need to inject the "classes_by_target" with # the compiled test java classes that JUnitRun will know which test # classes to execute. In a normal run, this "classes_by_target" will be # populated by java compiling step. class_products = context.products.get_data( 'classes_by_target', lambda: defaultdict(MultipleRootedProducts)) java_tests_products = MultipleRootedProducts() java_tests_products.add_rel_paths(test_abs_path, ['FooTest.class']) class_products[java_tests] = java_tests_products # Also we need to add the FooTest.class's classpath to the compile_classpath # products data mapping so JUnitRun will be able to add that into the final # classpath under which the junit will be executed. self.populate_compile_classpath(context=context, classpath=[test_abs_path]) # Finally execute the task. self.execute(context)
def execute_junit_runner(self, content, create_some_resources=True, **kwargs): # Create the temporary base test directory test_rel_path = 'tests/java/org/pantsbuild/foo' test_abs_path = self.create_dir(test_rel_path) # Generate the temporary java test source code. test_java_file_rel_path = os.path.join(test_rel_path, 'FooTest.java') test_java_file_abs_path = self.create_file(test_java_file_rel_path, content) # Create the temporary classes directory under work dir test_classes_abs_path = self.create_workdir_dir(test_rel_path) # Invoke ivy to resolve classpath for junit. classpath_file_abs_path = os.path.join(test_abs_path, 'junit.classpath') ivy_subsystem = global_subsystem_instance(IvySubsystem) distribution = DistributionLocator.cached(jdk=True) ivy = Bootstrapper(ivy_subsystem=ivy_subsystem).ivy() ivy.execute(args=['-cachepath', classpath_file_abs_path, '-dependency', 'junit', 'junit-dep', '4.10'], executor=SubprocessExecutor(distribution=distribution)) with open(classpath_file_abs_path) as fp: classpath = fp.read() # Now directly invoking javac to compile the test java code into java class # so later we can inject the class into products mapping for JUnitRun to execute # the test on. javac = distribution.binary('javac') subprocess.check_call( [javac, '-d', test_classes_abs_path, '-cp', classpath, test_java_file_abs_path]) # If a target_name is specified create a target with it, otherwise create a java_tests target. if 'target_name' in kwargs: target = self.target(kwargs['target_name']) else: target = self.create_library(test_rel_path, 'java_tests', 'foo_test', ['FooTest.java']) target_roots = [] if create_some_resources: # Create a synthetic resource target. target_roots.append(self.make_target('some_resources', Resources)) target_roots.append(target) # Set the context with the two targets, one java_tests target and # one synthetic resources target. # The synthetic resources target is to make sure we won't regress # in the future with bug like https://github.com/pantsbuild/pants/issues/508. Note # in that bug, the resources target must be the first one in the list. context = self.context(target_roots=target_roots) # Before we run the task, we need to inject the "runtime_classpath" with # the compiled test java classes that JUnitRun will know which test # classes to execute. In a normal run, this "runtime_classpath" will be # populated by java compilation step. self.populate_runtime_classpath(context=context, classpath=[test_classes_abs_path]) # Finally execute the task. self.execute(context)
def test_fresh_bootstrap(self): with temporary_dir() as fresh_bootstrap_dir: self.set_bootstrap_options(pants_bootstrapdir=fresh_bootstrap_dir) # Initialize the Ivy subsystem self.context() Bootstrapper.default_ivy() bootstrap_jar_path = os.path.join(fresh_bootstrap_dir, 'tools', 'jvm', 'ivy', 'bootstrap.jar') self.assertTrue(os.path.exists(bootstrap_jar_path))
def test_simple(self): ivy_subsystem = IvySubsystem.global_instance() bootstrapper = Bootstrapper(ivy_subsystem=ivy_subsystem) ivy = bootstrapper.ivy() self.assertIsNotNone(ivy.ivy_resolution_cache_dir) self.assertIsNone(ivy.ivy_settings) bootstrap_jar_path = os.path.join(ivy_subsystem.get_options().pants_bootstrapdir, 'tools', 'jvm', 'ivy', 'bootstrap.jar') self.assertTrue(os.path.exists(bootstrap_jar_path))
def test_simple(self): with subsystem_instance(IvySubsystem) as ivy_subsystem: bootstrapper = Bootstrapper(ivy_subsystem=ivy_subsystem) ivy = bootstrapper.ivy() self.assertIsNotNone(ivy.ivy_cache_dir) self.assertIsNone(ivy.ivy_settings) bootstrap_jar_path = os.path.join(ivy_subsystem.get_options().pants_bootstrapdir, 'tools', 'jvm', 'ivy', 'bootstrap.jar') self.assertTrue(os.path.exists(bootstrap_jar_path))
def test_simple(self): ivy_subsystem = IvySubsystem.global_instance() bootstrapper = Bootstrapper(ivy_subsystem=ivy_subsystem) ivy = bootstrapper.ivy() self.assertIsNotNone(ivy.ivy_cache_dir) self.assertIsNone(ivy.ivy_settings) bootstrap_jar_path = os.path.join( ivy_subsystem.get_options().pants_bootstrapdir, "tools", "jvm", "ivy", "bootstrap.jar" ) self.assertTrue(os.path.exists(bootstrap_jar_path))
def setUp(self): super(TaskTestBase, self).setUp() self._testing_task_type, self.options_scope = self.synthesize_task_subtype(self.task_type()) # We locate the workdir below the pants_workdir, which BaseTest locates within # the BuildRoot. self._tmpdir = tempfile.mkdtemp(dir=self.pants_workdir) self._test_workdir = os.path.join(self._tmpdir, 'workdir') os.mkdir(self._test_workdir) Bootstrapper.reset_instance() IvySubsystem.reset_global_instance()
def test_parse_proxy_string(self): bootstrapper = Bootstrapper().instance() self.assertEquals(('example.com', 1234), bootstrapper._parse_proxy_string('http://example.com:1234')) self.assertEquals(('secure-example.com', 999), bootstrapper._parse_proxy_string('http://secure-example.com:999')) # trailing slash is ok self.assertEquals(('example.com', 1234), bootstrapper._parse_proxy_string('http://example.com:1234/'))
def setUp(self): super(TaskTestBase, self).setUp() self._testing_task_type = self.synthesize_task_subtype(self.task_type(), self.options_scope) # We locate the workdir below the pants_workdir, which BaseTest locates within the BuildRoot. # BaseTest cleans this up, so we don't need to. We give it a stable name, so that we can # use artifact caching to speed up tests. self._test_workdir = os.path.join(self.pants_workdir, self.task_type().stable_name()) os.mkdir(self._test_workdir) # TODO: Push this down to JVM-related tests only? Seems wrong to have an ivy-specific # action in this non-JVM-specific, high-level base class. Bootstrapper.reset_instance()
def _temp_ivy_cache_dir(self): # We also reset the Bootstrapper since it hangs on to a ivy subsystem. old_instance = Bootstrapper._INSTANCE try: with temporary_dir() as ivy_cache_dir: Bootstrapper.reset_instance() # We set a temp ivy cache to ensure that the ivy we're using won't cache the temporary jar. self.set_options_for_scope('ivy', cache_dir=ivy_cache_dir) yield ivy_cache_dir finally: Bootstrapper._INSTANCE = old_instance
def setUp(self): super(TaskTestBase, self).setUp() self._testing_task_type, self.options_scope = self.synthesize_task_subtype(self.task_type()) # We locate the workdir below the pants_workdir, which BaseTest locates within # the BuildRoot. self._tmpdir = tempfile.mkdtemp(dir=self.pants_workdir) self._test_workdir = os.path.join(self._tmpdir, 'workdir') os.mkdir(self._test_workdir) # TODO: Push this down to JVM-related tests only? Seems wrong to have an ivy-specific # action in this non-JVM-specific, high-level base class. Bootstrapper.reset_instance()
def setUp(self): super(TaskTestBase, self).setUp() self.options_scope = 'test_scope' self._testing_task_type = self.synthesize_task_subtype(self.task_type(), self.options_scope) # We locate the workdir below the pants_workdir, which BaseTest locates within the BuildRoot. # BaseTest cleans this up, so we don't need to. We give it a stable name, so that we can # use artifact caching to speed up tests. self._test_workdir = os.path.join(self.pants_workdir, self.task_type().stable_name()) os.mkdir(self._test_workdir) # TODO: Push this down to JVM-related tests only? Seems wrong to have an ivy-specific # action in this non-JVM-specific, high-level base class. Bootstrapper.reset_instance()
def execute_junit_runner(self, content): # Create the temporary base test directory test_rel_path = "tests/java/org/pantsbuild/foo" test_abs_path = self.create_dir(test_rel_path) # Generate the temporary java test source code. test_java_file_rel_path = os.path.join(test_rel_path, "FooTest.java") test_java_file_abs_path = self.create_file(test_java_file_rel_path, content) # Create the temporary classes directory under work dir test_classes_abs_path = self.create_workdir_dir(test_rel_path) # Invoke ivy to resolve classpath for junit. classpath_file_abs_path = os.path.join(test_abs_path, "junit.classpath") with subsystem_instance(IvySubsystem) as ivy_subsystem: distribution = DistributionLocator.cached(jdk=True) ivy = Bootstrapper(ivy_subsystem=ivy_subsystem).ivy() ivy.execute( args=["-cachepath", classpath_file_abs_path, "-dependency", "junit", "junit-dep", "4.10"], executor=SubprocessExecutor(distribution=distribution), ) with open(classpath_file_abs_path) as fp: classpath = fp.read() # Now directly invoking javac to compile the test java code into java class # so later we can inject the class into products mapping for JUnitRun to execute # the test on. javac = distribution.binary("javac") subprocess.check_call([javac, "-d", test_classes_abs_path, "-cp", classpath, test_java_file_abs_path]) # Create a java_tests target and a synthetic resource target. java_tests = self.create_library(test_rel_path, "java_tests", "foo_test", ["FooTest.java"]) resources = self.make_target("some_resources", Resources) # Set the context with the two targets, one java_tests target and # one synthetic resources target. # The synthetic resources target is to make sure we won't regress # in the future with bug like https://github.com/pantsbuild/pants/issues/508. Note # in that bug, the resources target must be the first one in the list. context = self.context(target_roots=[resources, java_tests]) # Before we run the task, we need to inject the "runtime_classpath" with # the compiled test java classes that JUnitRun will know which test # classes to execute. In a normal run, this "runtime_classpath" will be # populated by java compilation step. self.populate_runtime_classpath(context=context, classpath=[test_classes_abs_path]) # Finally execute the task. self.execute(context)
def __init__(self, *args, **kwargs): super(IvyResolve, self).__init__(*args, **kwargs) self._ivy_bootstrapper = Bootstrapper.instance() self._cachedir = self._ivy_bootstrapper.ivy_cache_dir self._confs = self.context.config.getlist(self._CONFIG_SECTION, 'confs', default=['default']) self._classpath_dir = os.path.join(self.workdir, 'mapped') self._outdir = self.get_options().outdir or os.path.join( self.workdir, 'reports') self._open = self.get_options().open self._report = self._open or self.get_options().report self._ivy_bootstrap_key = 'ivy' self.register_jvm_tool_from_config(self._ivy_bootstrap_key, self.context.config, ini_section=self._CONFIG_SECTION, ini_key='bootstrap-tools', default=['//:xalan']) self._ivy_utils = IvyUtils(config=self.context.config, log=self.context.log) # Typically this should be a local cache only, since classpaths aren't portable. self.setup_artifact_cache_from_config( config_section=self._CONFIG_SECTION)
def dumped_chroot(self, targets): python_repos = create_subsystem(PythonRepos) with subsystem_instance(IvySubsystem) as ivy_subsystem: ivy_bootstrapper = Bootstrapper(ivy_subsystem=ivy_subsystem) with subsystem_instance( ThriftBinary.Factory) as thrift_binary_factory: interpreter_cache = PythonInterpreterCache( self.python_setup, python_repos) interpreter_cache.setup() interpreters = list( interpreter_cache.matches( [self.python_setup.interpreter_requirement])) self.assertGreater(len(interpreters), 0) interpreter = interpreters[0] with temporary_dir() as chroot: pex_builder = PEXBuilder(path=chroot, interpreter=interpreter) python_chroot = PythonChroot( python_setup=self.python_setup, python_repos=python_repos, ivy_bootstrapper=ivy_bootstrapper, thrift_binary_factory=thrift_binary_factory.create, interpreter=interpreter, builder=pex_builder, targets=targets, platforms=['current']) try: python_chroot.dump() yield pex_builder, python_chroot finally: python_chroot.delete()
def exec_ivy(cls, ivy, confs, ivyxml, args, jvm_options, executor, workunit_name, workunit_factory): ivy = ivy or Bootstrapper.default_ivy() ivy_args = ['-ivy', ivyxml] ivy_args.append('-confs') ivy_args.extend(confs) ivy_args.extend(args) ivy_jvm_options = list(jvm_options) # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng. ivy_jvm_options.append('-Dsun.io.useCanonCaches=false') runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor) try: result = execute_runner(runner, workunit_factory=workunit_factory, workunit_name=workunit_name) if result != 0: raise IvyUtils.IvyError( 'Ivy returned {result}. cmd={cmd}'.format(result=result, cmd=runner.cmd)) except runner.executor.Error as e: raise IvyUtils.IvyError(e)
def publish(self, ivyxml_path, jar, entry, repo, published): """Run ivy to publish a jar. ivyxml_path is the path to the ivy file; published is a list of jars published so far (including this one). entry is a pushdb entry.""" jvm_args = self._ivy_jvm_args(repo) resolver = repo['resolver'] path = repo.get('path') try: ivy = Bootstrapper.default_ivy() except Bootstrapper.Error as e: raise TaskError('Failed to push {0}! {1}'.format(pushdb_coordinate(jar, entry), e)) ivysettings = self.generate_ivysettings(ivy, published, publish_local=path) args = [ '-settings', ivysettings, '-ivy', ivyxml_path, '-deliverto', '%s/[organisation]/[module]/ivy-[revision].xml' % self.workdir, '-publish', resolver, '-publishpattern', '%s/[organisation]/[module]/' '[artifact]-[revision](-[classifier]).[ext]' % self.workdir, '-revision', entry.version().version(), '-m2compatible', ] if LogOptions.stderr_log_level() == logging.DEBUG: args.append('-verbose') if self.local_snapshot: args.append('-overwrite') try: ivy.execute(jvm_options=jvm_args, args=args, workunit_factory=self.context.new_workunit, workunit_name='jar-publish') except Ivy.Error as e: raise TaskError('Failed to push {0}! {1}'.format(pushdb_coordinate(jar, entry), e))
def publish(ivyxml_path): ivysettings = self.generate_ivysettings(published, publish_local=path) args = [ '-settings', ivysettings, '-ivy', ivyxml_path, '-deliverto', '%s/[organisation]/[module]/ivy-[revision].xml' % self.workdir, '-publish', resolver, '-publishpattern', '%s/[organisation]/[module]/' '[artifact]-[revision](-[classifier]).[ext]' % self.workdir, '-revision', newver.version(), '-m2compatible', ] if LogOptions.stderr_log_level() == logging.DEBUG: args.append('-verbose') if self.snapshot: args.append('-overwrite') try: ivy = Bootstrapper.default_ivy() ivy.execute(jvm_options=jvm_args, args=args, workunit_factory=self.context.new_workunit, workunit_name='jar-publish') except (Bootstrapper.Error, Ivy.Error) as e: raise TaskError('Failed to push %s! %s' % (jar_coordinate(jar, newver.version()), e))
def exec_ivy(cls, ivy, confs, ivyxml, args, jvm_options, executor, workunit_name, workunit_factory): """ :API: public """ ivy = ivy or Bootstrapper.default_ivy() ivy_args = ['-ivy', ivyxml] ivy_args.append('-confs') ivy_args.extend(confs) ivy_args.extend(args) ivy_jvm_options = list(jvm_options) # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng. ivy_jvm_options.append('-Dsun.io.useCanonCaches=false') runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor) try: with ivy.resolution_lock: result = execute_runner(runner, workunit_factory=workunit_factory, workunit_name=workunit_name) if result != 0: raise IvyUtils.IvyError('Ivy returned {result}. cmd={cmd}'.format(result=result, cmd=runner.cmd)) except runner.executor.Error as e: raise IvyUtils.IvyError(e)
def dumped_chroot(self, targets): # TODO(benjy): We shouldn't need to mention DistributionLocator here, as IvySubsystem # declares it as a dependency. However if we don't then test_antlr() below fails on # uninitialized options for that subsystem. Hopefully my pending (as of 9/2016) change # to clean up how we initialize and create instances of subsystems in tests will make # this problem go away. self.context(for_subsystems=[PythonRepos, PythonSetup, IvySubsystem, DistributionLocator, ThriftBinary.Factory, BinaryUtil.Factory]) python_repos = PythonRepos.global_instance() ivy_bootstrapper = Bootstrapper(ivy_subsystem=IvySubsystem.global_instance()) thrift_binary_factory = ThriftBinary.Factory.global_instance().create interpreter_cache = PythonInterpreterCache(self.python_setup, python_repos) interpreter = interpreter_cache.select_interpreter_for_targets(targets) self.assertIsNotNone(interpreter) with temporary_dir() as chroot: pex_builder = PEXBuilder(path=chroot, interpreter=interpreter) python_chroot = PythonChroot(python_setup=self.python_setup, python_repos=python_repos, ivy_bootstrapper=ivy_bootstrapper, thrift_binary_factory=thrift_binary_factory, interpreter=interpreter, builder=pex_builder, targets=targets, platforms=['current']) try: python_chroot.dump() yield pex_builder, python_chroot finally: python_chroot.delete()
def __init__(self, context, workdir, confs=None): super(IvyResolve, self).__init__(context, workdir) self._ivy_bootstrapper = Bootstrapper.instance() self._cachedir = self._ivy_bootstrapper.ivy_cache_dir self._confs = confs or context.config.getlist( 'ivy-resolve', 'confs', default=['default']) self._classpath_dir = os.path.join(self.workdir, 'mapped') self._outdir = context.options.ivy_resolve_outdir or os.path.join( self.workdir, 'reports') self._open = context.options.ivy_resolve_open self._report = self._open or context.options.ivy_resolve_report self._ivy_bootstrap_key = 'ivy' ivy_bootstrap_tools = context.config.getlist('ivy-resolve', 'bootstrap-tools', ':xalan') self.register_jvm_tool(self._ivy_bootstrap_key, ivy_bootstrap_tools) self._ivy_utils = IvyUtils(config=context.config, options=context.options, log=context.log) context.products.require_data('exclusives_groups') # Typically this should be a local cache only, since classpaths aren't portable. self.setup_artifact_cache_from_config(config_section='ivy-resolve')
def publish(ivyxml_path): try: ivy = Bootstrapper.default_ivy() except Bootstrapper.Error as e: raise TaskError('Failed to push %s! %s' % (jar_coordinate(jar, newver.version()), e)) ivysettings = self.generate_ivysettings(ivy, published, publish_local=path) args = [ '-settings', ivysettings, '-ivy', ivyxml_path, '-deliverto', '%s/[organisation]/[module]/ivy-[revision].xml' % self.workdir, '-publish', resolver, '-publishpattern', '%s/[organisation]/[module]/' '[artifact]-[revision](-[classifier]).[ext]' % self.workdir, '-revision', newver.version(), '-m2compatible', ] if LogOptions.stderr_log_level() == logging.DEBUG: args.append('-verbose') if self.snapshot: args.append('-overwrite') try: ivy.execute(jvm_options=jvm_args, args=args, workunit_factory=self.context.new_workunit, workunit_name='jar-publish') except Ivy.Error as e: raise TaskError('Failed to push %s! %s' % (jar_coordinate(jar, newver.version()), e))
def setUp(self): """ :API: public """ super().setUp() # Use a synthetic subclass for proper isolation when bootstrapping within the test. bootstrap_scope = "bootstrap_scope" self.bootstrap_task_type = self.synthesize_task_subtype( BootstrapJvmTools, bootstrap_scope) JvmToolMixin.reset_registered_tools() # Set some options: # 1. Cap BootstrapJvmTools memory usage in tests. The Xmx was empirically arrived upon using # -Xloggc and verifying no full gcs for a test using the full gamut of resolving a multi-jar # tool, constructing a fat jar and then shading that fat jar. # # 2. Allow tests to read/write tool jars from the real artifact cache, so they don't # each have to resolve and shade them every single time, which is a huge slowdown. # Note that local artifact cache writes are atomic, so it's fine for multiple concurrent # tests to write to it. # # Note that we don't have access to the invoking pants instance's options, so we assume that # its artifact cache is in the standard location. If it isn't, worst case the tests will # populate a second cache at the standard location, which is no big deal. # TODO: We really need a straightforward way for pants's own tests to get to the enclosing # pants instance's options values. artifact_caches = [ os.path.join(get_pants_cachedir(), "artifact_cache") ] self.set_options_for_scope( bootstrap_scope, execution_strategy=NailgunTask.ExecutionStrategy.subprocess, jvm_options=["-Xmx128m"], ) self.set_options_for_scope(f"cache.{bootstrap_scope}", read_from=artifact_caches, write_to=artifact_caches) # Copy into synthetic build-root shutil.copy("BUILD.tools", self.build_root) build_root_third_party = os.path.join(self.build_root, "3rdparty") safe_mkdir(build_root_third_party) shutil.copy(os.path.join("3rdparty", "BUILD"), build_root_third_party) Bootstrapper.reset_instance()
def exec_ivy(self, target_workdir, targets, args, confs=None, ivy=None, workunit_name='ivy', workunit_factory=None, symlink_ivyxml=False, jars=None): ivy = ivy or Bootstrapper.default_ivy() if not isinstance(ivy, Ivy): raise ValueError('The ivy argument supplied must be an Ivy instance, given %s of type %s' % (ivy, type(ivy))) ivyxml = os.path.join(target_workdir, 'ivy.xml') if not jars: jars, excludes = self._calculate_classpath(targets) else: excludes = set() ivy_args = ['-ivy', ivyxml] confs_to_resolve = confs or ['default'] ivy_args.append('-confs') ivy_args.extend(confs_to_resolve) ivy_args.extend(args) if not self._transitive: ivy_args.append('-notransitive') ivy_args.extend(self._args) def safe_link(src, dest): try: os.unlink(dest) except OSError as e: if e.errno != errno.ENOENT: raise os.symlink(src, dest) with IvyUtils.ivy_lock: self._generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve) runner = ivy.runner(jvm_options=self._jvm_options, args=ivy_args) try: result = util.execute_runner(runner, workunit_factory=workunit_factory, workunit_name=workunit_name) # Symlink to the current ivy.xml file (useful for IDEs that read it). if symlink_ivyxml: ivyxml_symlink = os.path.join(self._workdir, 'ivy.xml') safe_link(ivyxml, ivyxml_symlink) if result != 0: raise TaskError('Ivy returned %d' % result) except runner.executor.Error as e: raise TaskError(e)
def test_depmap_jar_path(self): with temporary_dir(root_dir=self.workdir_root()) as workdir: test_target = 'examples/tests/java/com/pants/examples/usethrift:usethrift' json_data = self.run_depmap_project_info(test_target, workdir) ivy_cache_dir = Bootstrapper.instance().ivy_cache_dir self.assertEquals(json_data.get('libraries').get('commons-lang:commons-lang:2.5'), [os.path.join(ivy_cache_dir, 'commons-lang/commons-lang/jars/commons-lang-2.5.jar')])
def test_simple(self): bootstrapper = Bootstrapper.instance() ivy = bootstrapper.ivy() self.assertIsNotNone(ivy.ivy_cache_dir) self.assertIsNone(ivy.ivy_settings) bootstrap_jar_path = os.path.join(self.test_workdir, 'tools', 'jvm', 'ivy', 'bootstrap.jar') self.assertTrue(os.path.exists(bootstrap_jar_path))
def publish(self, publications, jar, entry, repo, published): """Run ivy to publish a jar. ivyxml_path is the path to the ivy file; published is a list of jars published so far (including this one). entry is a pushdb entry.""" try: ivy = Bootstrapper.default_ivy() except Bootstrapper.Error as e: raise TaskError('Failed to push {0}! {1}'.format( pushdb_coordinate(jar, entry), e)) path = repo.get('path') ivysettings = self.generate_ivysettings(ivy, published, publish_local=path) version = entry.version().version() ivyxml = self.generate_ivy(jar, version, publications) resolver = repo['resolver'] args = [ '-settings', ivysettings, '-ivy', ivyxml, # Without this setting, the ivy.xml is delivered to the CWD, littering the workspace. We # don't need the ivy.xml, so just give it path under the workdir we won't use. '-deliverto', ivyxml + '.unused', '-publish', resolver, '-publishpattern', '{}/[organisation]/[module]/' '[artifact]-[revision](-[classifier]).[ext]'.format(self.workdir), '-revision', version, '-m2compatible', ] # TODO(John Sirois): global logging options should be hidden behind some sort of log manager # that we can: # a.) obtain a handle to (dependency injection or manual plumbing) # b.) query for log detail, ie: `if log_manager.is_verbose:` if self.get_options().level == 'debug': args.append('-verbose') if self.local_snapshot: args.append('-overwrite') try: jvm_options = self._ivy_jvm_options(repo) ivy.execute(jvm_options=jvm_options, args=args, workunit_factory=self.context.new_workunit, workunit_name='ivy-publish') except Ivy.Error as e: raise TaskError('Failed to push {0}! {1}'.format( pushdb_coordinate(jar, entry), e))
def setUp(self): super(JvmToolTaskTestBase, self).setUp() # Use a synthetic subclass for proper isolation when bootstrapping within the test. bootstrap_scope = 'bootstrap_scope' self.bootstrap_task_type = self.synthesize_task_subtype(BootstrapJvmTools, bootstrap_scope) JvmToolMixin.reset_registered_tools() # Cap BootstrapJvmTools memory usage in tests. The Xmx was empirically arrived upon using # -Xloggc and verifying no full gcs for a test using the full gamut of resolving a multi-jar # tool, constructing a fat jar and then shading that fat jar. self.set_options_for_scope(bootstrap_scope, jvm_options=['-Xmx128m']) # Tool option defaults currently point to targets in the real BUILD.tools, so we copy it # into our test workspace. shutil.copy(os.path.join(self.real_build_root, 'BUILD.tools'), self.build_root) Bootstrapper.reset_instance()
def test_depmap_jar_path(self): with temporary_dir(root_dir=self.workdir_root()) as workdir: test_target = 'examples/tests/java/com/pants/examples/usethrift:usethrift' json_data = self.run_depmap_project_info(test_target, workdir) # Hack because Bootstrapper.instance() reads config from cache. Will go away after we plumb # options into IvyUtil properly. Config.cache(Config.load()) ivy_cache_dir = Bootstrapper.instance().ivy_cache_dir self.assertEquals(json_data.get('libraries').get('commons-lang:commons-lang:2.5'), [os.path.join(ivy_cache_dir, 'commons-lang/commons-lang/jars/commons-lang-2.5.jar')])
def publish(self, publications, jar, entry, repo, published): """Run ivy to publish a jar. ivyxml_path is the path to the ivy file; published is a list of jars published so far (including this one). entry is a pushdb entry.""" try: ivy = Bootstrapper.default_ivy() except Bootstrapper.Error as e: raise TaskError("Failed to push {0}! {1}".format(pushdb_coordinate(jar, entry), e)) path = repo.get("path") ivysettings = self.generate_ivysettings(ivy, published, publish_local=path) version = entry.version().version() ivyxml = self.generate_ivy(jar, version, publications) resolver = repo["resolver"] args = [ "-settings", ivysettings, "-ivy", ivyxml, # Without this setting, the ivy.xml is delivered to the CWD, littering the workspace. We # don't need the ivy.xml, so just give it path under the workdir we won't use. "-deliverto", ivyxml + ".unused", "-publish", resolver, "-publishpattern", "{}/[organisation]/[module]/" "[artifact]-[revision](-[classifier]).[ext]".format(self.workdir), "-revision", version, "-m2compatible", ] # TODO(John Sirois): global logging options should be hidden behind some sort of log manager # that we can: # a.) obtain a handle to (dependency injection or manual plumbing) # b.) query for log detail, ie: `if log_manager.is_verbose:` if self.get_options().level == "debug": args.append("-verbose") if self.local_snapshot: args.append("-overwrite") try: jvm_options = self._ivy_jvm_options(repo) ivy.execute( jvm_options=jvm_options, args=args, workunit_factory=self.context.new_workunit, workunit_name="ivy-publish", ) except Ivy.Error as e: raise TaskError("Failed to push {0}! {1}".format(pushdb_coordinate(jar, entry), e))
def mapjars(self, genmap, target, executor, workunit_factory=None, jars=None): """Resolves jars for the target and stores their locations in genmap. :param genmap: The jar_dependencies ProductMapping entry for the required products. :param target: The target whose jar dependencies are being retrieved. :param jars: If specified, resolves the given jars rather than :type jars: List of :class:`pants.backend.jvm.targets.jar_dependency.JarDependency` (jar()) objects. """ mapdir = os.path.join(self.mapto_dir(), target.id) safe_mkdir(mapdir, clean=True) ivyargs = [ '-retrieve', '%s/[organisation]/[artifact]/[conf]/' '[organisation]-[artifact]-[revision](-[classifier]).[ext]' % mapdir, '-symlink', ] self.exec_ivy(mapdir, [target], ivyargs, confs=target.payload.configurations, ivy=Bootstrapper.default_ivy(executor), workunit_factory=workunit_factory, workunit_name='map-jars', jars=jars) for org in os.listdir(mapdir): orgdir = os.path.join(mapdir, org) if os.path.isdir(orgdir): for name in os.listdir(orgdir): artifactdir = os.path.join(orgdir, name) if os.path.isdir(artifactdir): for conf in os.listdir(artifactdir): confdir = os.path.join(artifactdir, conf) for f in os.listdir(confdir): if self.is_mappable_artifact(org, name, f): # TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a # conf whitelist genmap.add(org, confdir).append(f) genmap.add((org, name), confdir).append(f) genmap.add(target, confdir).append(f) genmap.add((target, conf), confdir).append(f) genmap.add((org, name, conf), confdir).append(f)
def publish(self, publications, jar, version, repo, published): """Run ivy to publish a jar. :param str ivyxml_path: The path to the ivy file. :param list published: A list of jars published so far (including this one). """ try: ivy = Bootstrapper.default_ivy() except Bootstrapper.Error as e: raise TaskError('Failed to push {0}! {1}'.format(jar, e)) if ivy.ivy_settings is None: raise TaskError('An ivysettings.xml with writeable resolvers is required for publishing, ' 'but none was configured.') path = repo.get('path') ivysettings = self.generate_ivysettings(self.fetch_ivysettings(ivy), published, publish_local=path) ivyxml = self.generate_ivy(jar, version, publications) resolver = repo['resolver'] args = [ '-settings', ivysettings, '-ivy', ivyxml, # Without this setting, the ivy.xml is delivered to the CWD, littering the workspace. We # don't need the ivy.xml, so just give it path under the workdir we won't use. '-deliverto', ivyxml + '.unused', '-publish', resolver, '-publishpattern', '{}/[organisation]/[module]/' '[artifact]-[revision](-[classifier]).[ext]'.format(self.workdir), '-revision', version, '-m2compatible', ] if self.get_options().level == 'debug': args.append('-verbose') if self.get_options().local: args.append('-overwrite') if not self.get_options().dryrun: try: ivy.execute(jvm_options=self._ivy_jvm_options(repo), args=args, workunit_factory=self.context.new_workunit, workunit_name='ivy-publish') except Ivy.Error as e: raise TaskError('Failed to push {0}! {1}'.format(jar, e)) else: print("\nDRYRUN- would have pushed:{} using {} resolver.".format(self.jar_coordinate(jar, version), resolver))
def _exec_ivy(self, target_workdir, targets, args, executor=None, confs=None, ivy=None, workunit_name='ivy', use_soft_excludes=False, resolve_hash_name=None): ivy_jvm_options = self.get_options().jvm_options[:] # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng. ivy_jvm_options.append('-Dsun.io.useCanonCaches=false') ivy = ivy or Bootstrapper.default_ivy() ivyxml = os.path.join(target_workdir, 'ivy.xml') ivy_args = ['-ivy', ivyxml] confs_to_resolve = confs or ('default', ) ivy_args.append('-confs') ivy_args.extend(confs_to_resolve) ivy_args.extend(args) # TODO(John Sirois): merge the code below into IvyUtils or up here; either way, better # diagnostics can be had in `IvyUtils.generate_ivy` if this is done. # See: https://github.com/pantsbuild/pants/issues/2239 try: jars, excludes = IvyUtils.calculate_classpath( targets, gather_excludes=not use_soft_excludes) with IvyUtils.ivy_lock: IvyUtils.generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve, resolve_hash_name) runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor) try: result = execute_runner( runner, workunit_factory=self.context.new_workunit, workunit_name=workunit_name) if result != 0: raise self.Error( 'Ivy returned {result}. cmd={cmd}'.format( result=result, cmd=runner.cmd)) except runner.executor.Error as e: raise self.Error(e) except IvyUtils.IvyError as e: raise self.Error('Failed to prepare ivy resolve: {}'.format(e))
def test_depmap_jar_path(self): with temporary_dir(root_dir=self.workdir_root()) as workdir: test_target = 'examples/tests/java/com/pants/examples/usethrift:usethrift' json_data = self.run_depmap_project_info(test_target, workdir) # Hack because Bootstrapper.instance() reads config from cache. Will go away after we plumb # options into IvyUtil properly. Config.cache(Config.load()) ivy_cache_dir = Bootstrapper.instance().ivy_cache_dir self.assertEquals( json_data.get('libraries').get( 'commons-lang:commons-lang:2.5'), [ os.path.join( ivy_cache_dir, 'commons-lang/commons-lang/jars/commons-lang-2.5.jar') ])
def __init__(self, *args, **kwargs): super(IvyResolve, self).__init__(*args, **kwargs) self._ivy_bootstrapper = Bootstrapper.instance() self._cachedir = self._ivy_bootstrapper.ivy_cache_dir self._confs = self.context.config.getlist(self._CONFIG_SECTION, 'confs', default=['default']) self._classpath_dir = os.path.join(self.workdir, 'mapped') self._outdir = self.get_options().outdir or os.path.join(self.workdir, 'reports') self._open = self.get_options().open self._report = self._open or self.get_options().report self._ivy_utils = IvyUtils(config=self.context.config, log=self.context.log) # Typically this should be a local cache only, since classpaths aren't portable. self.setup_artifact_cache_from_config(config_section=self._CONFIG_SECTION)
def exec_ivy(self, target_workdir, targets, args, executor=None, confs=None, ivy=None, workunit_name='ivy', jars=None, use_soft_excludes=False, resolve_hash_name=None): ivy_jvm_options = copy.copy(self.get_options().jvm_options) # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng. ivy_jvm_options.append('-Dsun.io.useCanonCaches=false') ivy = ivy or Bootstrapper.default_ivy() ivyxml = os.path.join(target_workdir, 'ivy.xml') if not jars: jars, excludes = IvyUtils.calculate_classpath( targets, gather_excludes=not use_soft_excludes) else: excludes = set() ivy_args = ['-ivy', ivyxml] confs_to_resolve = confs or ['default'] ivy_args.append('-confs') ivy_args.extend(confs_to_resolve) ivy_args.extend(args) with IvyUtils.ivy_lock: IvyUtils.generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve, resolve_hash_name) runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor) try: result = execute_runner( runner, workunit_factory=self.context.new_workunit, workunit_name=workunit_name) if result != 0: raise TaskError('Ivy returned {result}. cmd={cmd}'.format( result=result, cmd=runner.cmd)) except runner.executor.Error as e: raise TaskError(e)
def __init__(self, *args, **kwargs): super(IvyResolve, self).__init__(*args, **kwargs) self._ivy_bootstrapper = Bootstrapper.instance() self._cachedir = self._ivy_bootstrapper.ivy_cache_dir self._classpath_dir = os.path.join(self.workdir, 'mapped') self._outdir = self.get_options().outdir or os.path.join(self.workdir, 'reports') self._open = self.get_options().open self._report = self._open or self.get_options().report self._confs = None self._args = [] for arg in self.get_options().args: self._args.extend(safe_shlex_split(arg)) # Typically this should be a local cache only, since classpaths aren't portable. self.setup_artifact_cache()
def run_antlrs(self, output_dir): # TODO(John Sirois): graduate to a JvmToolTask and/or merge with the java code gen AntlrGen # task. args = [ '-dependency', 'org.antlr', 'antlr', self.target.antlr_version, '-types', 'jar', '-main', 'org.antlr.Tool', '--', '-fo', output_dir ] for source in self.target.sources_relative_to_buildroot(): abs_path = os.path.join(get_buildroot(), source) args.append(abs_path) try: ivy = Bootstrapper.default_ivy() ivy.execute( args=args ) # TODO: Needs a workunit, when we have a context here. except (Bootstrapper.Error, Ivy.Error) as e: raise TaskError('ANTLR generation failed! {0}'.format(e))
def run_antlrs(self, output_dir): # TODO(John Sirois): graduate to a JvmToolTask and/or merge with the java code gen AntlrGen # task. args = [ '-dependency', 'org.antlr', 'antlr', self.target.antlr_version, '-types', 'jar', '-main', 'org.antlr.Tool', '--', '-fo', output_dir ] for source in self.target.sources_relative_to_buildroot(): abs_path = os.path.join(get_buildroot(), source) args.append(abs_path) try: ivy = Bootstrapper.default_ivy() ivy.execute(args=args) # TODO: Needs a workunit, when we have a context here. except (Bootstrapper.Error, Ivy.Error) as e: raise TaskError('ANTLR generation failed! {0}'.format(e))
def run_antlrs(self, output_dir): args = [ '-dependency', 'org.antlr', 'antlr', self.target.antlr_version, '-types', 'jar', '-main', 'org.antlr.Tool', '--', '-fo', output_dir ] for source in self.target.sources_relative_to_buildroot(): abs_path = os.path.join(get_buildroot(), source) args.append(abs_path) try: ivy = Bootstrapper.default_ivy() ivy.execute( args=args ) # TODO: Needs a workunit, when we have a context here. return True except (Bootstrapper.Error, Ivy.Error) as e: print('ANTLR generation failed! %s' % e, file=sys.stderr) return False
def __init__(self, *args, **kwargs): super(IvyResolve, self).__init__(*args, **kwargs) self._ivy_bootstrapper = Bootstrapper.instance() self._cachedir = self._ivy_bootstrapper.ivy_cache_dir self._classpath_dir = os.path.join(self.workdir, 'mapped') self._outdir = self.get_options().outdir or os.path.join( self.workdir, 'reports') self._open = self.get_options().open self._report = self._open or self.get_options().report self._confs = None self._args = [] for arg in self.get_options().args: self._args.extend(safe_shlex_split(arg)) # Typically this should be a local cache only, since classpaths aren't portable. self.setup_artifact_cache()
def _exec_ivy( self, target_workdir, targets, args, executor=None, confs=None, ivy=None, workunit_name="ivy", use_soft_excludes=False, resolve_hash_name=None, ): ivy_jvm_options = self.get_options().jvm_options[:] # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng. ivy_jvm_options.append("-Dsun.io.useCanonCaches=false") ivy = ivy or Bootstrapper.default_ivy() ivyxml = os.path.join(target_workdir, "ivy.xml") ivy_args = ["-ivy", ivyxml] confs_to_resolve = confs or ("default",) ivy_args.append("-confs") ivy_args.extend(confs_to_resolve) ivy_args.extend(args) # TODO(John Sirois): merge the code below into IvyUtils or up here; either way, better # diagnostics can be had in `IvyUtils.generate_ivy` if this is done. # See: https://github.com/pantsbuild/pants/issues/2239 try: jars, excludes = IvyUtils.calculate_classpath(targets, gather_excludes=not use_soft_excludes) with IvyUtils.ivy_lock: IvyUtils.generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve, resolve_hash_name) runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor) try: result = execute_runner( runner, workunit_factory=self.context.new_workunit, workunit_name=workunit_name ) if result != 0: raise self.Error("Ivy returned {result}. cmd={cmd}".format(result=result, cmd=runner.cmd)) except runner.executor.Error as e: raise self.Error(e) except IvyUtils.IvyError as e: raise self.Error("Failed to prepare ivy resolve: {}".format(e))
def mapjars(self, genmap, target, executor, jars=None): """Resolves jars for the target and stores their locations in genmap. :param genmap: The jar_dependencies ProductMapping entry for the required products. :param target: The target whose jar dependencies are being retrieved. :param jars: If specified, resolves the given jars rather than :type jars: List of :class:`pants.backend.jvm.targets.jar_dependency.JarDependency` (jar()) objects. """ mapdir = os.path.join(self.workdir, 'mapped-jars', target.id) safe_mkdir(mapdir, clean=True) ivyargs = [ '-retrieve', '%s/[organisation]/[artifact]/[conf]/' '[organisation]-[artifact]-[revision](-[classifier]).[ext]' % mapdir, '-symlink', ] confs = maybe_list(target.payload.get_field_value('configurations') or []) self.exec_ivy(mapdir, [target], executor=executor, args=ivyargs, confs=confs, ivy=Bootstrapper.default_ivy(), workunit_name='map-jars', jars=jars) for org in os.listdir(mapdir): orgdir = os.path.join(mapdir, org) if os.path.isdir(orgdir): for name in os.listdir(orgdir): artifactdir = os.path.join(orgdir, name) if os.path.isdir(artifactdir): for conf in os.listdir(artifactdir): confdir = os.path.join(artifactdir, conf) for f in os.listdir(confdir): # TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a # conf whitelist genmap.add(org, confdir).append(f) genmap.add((org, name), confdir).append(f) genmap.add(target, confdir).append(f) genmap.add((target, conf), confdir).append(f) genmap.add((org, name, conf), confdir).append(f)
def exec_ivy(self, target_workdir, targets, args, executor=None, confs=None, ivy=None, workunit_name='ivy', jars=None, use_soft_excludes=False, resolve_hash_name=None): ivy_jvm_options = copy.copy(self.get_options().jvm_options) # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng. ivy_jvm_options.append('-Dsun.io.useCanonCaches=false') ivy = ivy or Bootstrapper.default_ivy() ivyxml = os.path.join(target_workdir, 'ivy.xml') if not jars: automatic_excludes = self.get_options().automatic_excludes jars, excludes = IvyUtils.calculate_classpath(targets, gather_excludes=not use_soft_excludes, automatic_excludes=automatic_excludes) else: excludes = set() ivy_args = ['-ivy', ivyxml] confs_to_resolve = confs or ['default'] ivy_args.append('-confs') ivy_args.extend(confs_to_resolve) ivy_args.extend(args) with IvyUtils.ivy_lock: IvyUtils.generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve, resolve_hash_name) runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor) try: result = execute_runner(runner, workunit_factory=self.context.new_workunit, workunit_name=workunit_name) if result != 0: raise TaskError('Ivy returned {result}. cmd={cmd}'.format(result=result, cmd=runner.cmd)) except runner.executor.Error as e: raise TaskError(e)
def do_resolve(cls, executor, extra_args, ivyxml, jvm_options, workdir_report_paths_by_conf, confs, ivy_cache_dir, ivy_cache_classpath_filename, resolve_hash_name, workunit_factory, workunit_name): """Execute Ivy with the given ivy.xml and copies all relevant files into the workdir. This method does an Ivy resolve, which may be either a Pants resolve or a Pants fetch depending on whether there is an existing frozen resolution. After it is run, the Ivy reports are copied into the workdir at the paths specified by workdir_report_paths_by_conf along with a file containing a list of all the requested artifacts and their transitive dependencies. :param executor: A JVM executor to use to invoke ivy. :param extra_args: Extra arguments to pass to ivy. :param ivyxml: The input ivy.xml containing the dependencies to resolve. :param jvm_options: A list of jvm option strings to use for the ivy invoke, or None. :param workdir_report_paths_by_conf: A dict mapping confs to report paths in the workdir. :param confs: The confs used in the resolve. :param resolve_hash_name: The hash to use as the module name for finding the ivy report file. :param workunit_factory: A workunit factory for the ivy invoke, or None. :param workunit_name: A workunit name for the ivy invoke, or None. """ ivy = Bootstrapper.default_ivy(bootstrap_workunit_factory=workunit_factory) with safe_concurrent_creation(ivy_cache_classpath_filename) as raw_target_classpath_file_tmp: extra_args = extra_args or [] args = ['-cachepath', raw_target_classpath_file_tmp] + extra_args with cls._ivy_lock: cls._exec_ivy(ivy, confs, ivyxml, args, jvm_options=jvm_options, executor=executor, workunit_name=workunit_name, workunit_factory=workunit_factory) if not os.path.exists(raw_target_classpath_file_tmp): raise cls.IvyError('Ivy failed to create classpath file at {}' .format(raw_target_classpath_file_tmp)) cls._copy_ivy_reports(workdir_report_paths_by_conf, confs, ivy_cache_dir, resolve_hash_name) logger.debug('Moved ivy classfile file to {dest}' .format(dest=ivy_cache_classpath_filename))
def mapjars(self, genmap, target, executor, workunit_factory=None): """ Parameters: genmap: the jar_dependencies ProductMapping entry for the required products. target: the target whose jar dependencies are being retrieved. """ mapdir = os.path.join(self.mapto_dir(), target.id) safe_mkdir(mapdir, clean=True) ivyargs = [ '-retrieve', '%s/[organisation]/[artifact]/[conf]/' '[organisation]-[artifact]-[revision](-[classifier]).[ext]' % mapdir, '-symlink', ] self.exec_ivy(mapdir, [target], ivyargs, confs=target.payload.configurations, ivy=Bootstrapper.default_ivy(executor), workunit_factory=workunit_factory, workunit_name='map-jars') for org in os.listdir(mapdir): orgdir = os.path.join(mapdir, org) if os.path.isdir(orgdir): for name in os.listdir(orgdir): artifactdir = os.path.join(orgdir, name) if os.path.isdir(artifactdir): for conf in os.listdir(artifactdir): confdir = os.path.join(artifactdir, conf) for f in os.listdir(confdir): if self.is_mappable_artifact(org, name, f): # TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a # conf whitelist genmap.add(org, confdir).append(f) genmap.add((org, name), confdir).append(f) genmap.add(target, confdir).append(f) genmap.add((target, conf), confdir).append(f) genmap.add((org, name, conf), confdir).append(f)
def publish(self, ivyxml_path, jar, entry, repo, published): """Run ivy to publish a jar. ivyxml_path is the path to the ivy file; published is a list of jars published so far (including this one). entry is a pushdb entry.""" jvm_options = self._ivy_jvm_options(repo) resolver = repo['resolver'] path = repo.get('path') try: ivy = Bootstrapper.default_ivy() except Bootstrapper.Error as e: raise TaskError('Failed to push {0}! {1}'.format(pushdb_coordinate(jar, entry), e)) ivysettings = self.generate_ivysettings(ivy, published, publish_local=path) args = [ '-settings', ivysettings, '-ivy', ivyxml_path, '-deliverto', '%s/[organisation]/[module]/ivy-[revision].xml' % self.workdir, '-publish', resolver, '-publishpattern', '%s/[organisation]/[module]/' '[artifact]-[revision](-[classifier]).[ext]' % self.workdir, '-revision', entry.version().version(), '-m2compatible', ] # TODO(John Sirois): global logging options should be hidden behind some sort of log manager # that we can: # a.) obtain a handle to (dependency injection or manual plumbing) # b.) query for log detail, ie: `if log_manager.is_verbose:` if self.get_options().level == 'debug': args.append('-verbose') if self.local_snapshot: args.append('-overwrite') try: ivy.execute(jvm_options=jvm_options, args=args, workunit_factory=self.context.new_workunit, workunit_name='jar-publish') except Ivy.Error as e: raise TaskError('Failed to push {0}! {1}'.format(pushdb_coordinate(jar, entry), e))
def test_default_ivy(self): ivy = Bootstrapper.default_ivy() self.assertIsNotNone(ivy.ivy_cache_dir) self.assertIsNone(ivy.ivy_settings)
def test_reset(self): bootstrapper1 = Bootstrapper.instance() Bootstrapper.reset_instance() bootstrapper2 = Bootstrapper.instance() self.assertNotEqual(bootstrapper1, bootstrapper2)
def ivy_resolve(self, targets, executor=None, silent=False, workunit_name=None, confs=None, custom_args=None): """Executes an ivy resolve for the relevant subset of the given targets. :returns: the resulting classpath, and the unique part of the name used for the resolution report (a hash). Also populates the 'ivy_resolve_symlink_map' product for jars resulting from the resolve.""" if not targets: return ([], None) ivy = Bootstrapper.default_ivy(bootstrap_workunit_factory=self.context.new_workunit) ivy_workdir = os.path.join(self.context.options.for_global_scope().pants_workdir, 'ivy') fingerprint_strategy = IvyResolveFingerprintStrategy(confs) with self.invalidated(targets, invalidate_dependents=False, silent=silent, fingerprint_strategy=fingerprint_strategy) as invalidation_check: if not invalidation_check.all_vts: return ([], None) global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts) # If a report file is not present, we need to exec ivy, even if all the individual # targets up to date... See https://rbcommons.com/s/twitter/r/2015 report_missing = False report_confs = confs or ['default'] report_paths = [] resolve_hash_name = global_vts.cache_key.hash for conf in report_confs: report_path = IvyUtils.xml_report_path(resolve_hash_name, conf) if not os.path.exists(report_path): report_missing = True break else: report_paths.append(report_path) target_workdir = os.path.join(ivy_workdir, resolve_hash_name) target_classpath_file = os.path.join(target_workdir, 'classpath') raw_target_classpath_file = target_classpath_file + '.raw' raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp' # A common dir for symlinks into the ivy2 cache. This ensures that paths to jars # in artifact-cached analysis files are consistent across systems. # Note that we have one global, well-known symlink dir, again so that paths are # consistent across builds. symlink_dir = os.path.join(ivy_workdir, 'jars') # Note that it's possible for all targets to be valid but for no classpath file to exist at # target_classpath_file, e.g., if we previously built a superset of targets. if report_missing or invalidation_check.invalid_vts or not os.path.exists(raw_target_classpath_file): args = ['-cachepath', raw_target_classpath_file_tmp] + (custom_args if custom_args else []) self.exec_ivy( target_workdir=target_workdir, targets=global_vts.targets, args=args, executor=executor, ivy=ivy, workunit_name=workunit_name, confs=confs, use_soft_excludes=self.get_options().soft_excludes, resolve_hash_name=resolve_hash_name) if not os.path.exists(raw_target_classpath_file_tmp): raise TaskError('Ivy failed to create classpath file at {}' .format(raw_target_classpath_file_tmp)) shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file) logger.debug('Moved ivy classfile file to {dest}'.format(dest=raw_target_classpath_file)) if self.artifact_cache_writes_enabled(): self.update_artifact_cache([(global_vts, [raw_target_classpath_file])]) else: logger.debug("Using previously resolved reports: {}".format(report_paths)) # Make our actual classpath be symlinks, so that the paths are uniform across systems. # Note that we must do this even if we read the raw_target_classpath_file from the artifact # cache. If we cache the target_classpath_file we won't know how to create the symlinks. with IvyTaskMixin.symlink_map_lock: products = self.context.products existing_symlinks_map = products.get_data('ivy_resolve_symlink_map', lambda: dict()) symlink_map = IvyUtils.symlink_cachepath( IvySubsystem.global_instance().get_options().cache_dir, raw_target_classpath_file, symlink_dir, target_classpath_file, existing_symlinks_map) existing_symlinks_map.update(symlink_map) with IvyUtils.cachepath(target_classpath_file) as classpath: stripped_classpath = [path.strip() for path in classpath] return (stripped_classpath, resolve_hash_name)
def test_simple(self): ivy_subsystem = IvySubsystem.global_instance() bootstrapper = Bootstrapper(ivy_subsystem=ivy_subsystem) ivy = bootstrapper.ivy() self.assertIsNotNone(ivy.ivy_resolution_cache_dir) self.assertIsNone(ivy.ivy_settings)
def _execute_junit_runner(self, list_of_filename_content_tuples, create_some_resources=True, target_name=None): # Create the temporary base test directory test_rel_path = 'tests/java/org/pantsbuild/foo' test_abs_path = self.create_dir(test_rel_path) # Create the temporary classes directory under work dir test_classes_abs_path = self.create_workdir_dir(test_rel_path) test_java_file_abs_paths = [] # Generate the temporary java test source code. for filename, content in list_of_filename_content_tuples: test_java_file_rel_path = os.path.join(test_rel_path, filename) test_java_file_abs_path = self.create_file(test_java_file_rel_path, content) test_java_file_abs_paths.append(test_java_file_abs_path) # Invoke ivy to resolve classpath for junit. classpath_file_abs_path = os.path.join(test_abs_path, 'junit.classpath') ivy_subsystem = global_subsystem_instance(IvySubsystem) distribution = DistributionLocator.cached(jdk=True) ivy = Bootstrapper(ivy_subsystem=ivy_subsystem).ivy() ivy.execute(args=[ '-cachepath', classpath_file_abs_path, '-dependency', 'junit', 'junit-dep', '4.10' ], executor=SubprocessExecutor(distribution=distribution)) with open(classpath_file_abs_path) as fp: classpath = fp.read() # Now directly invoke javac to compile the test java code into classfiles that we can later # inject into a product mapping for JUnitRun to execute against. javac = distribution.binary('javac') subprocess.check_call( [javac, '-d', test_classes_abs_path, '-cp', classpath] + test_java_file_abs_paths) # If a target_name is specified create a target with it, otherwise create a junit_tests target. if target_name: target = self.target(target_name) else: target = self.create_library(test_rel_path, 'junit_tests', 'foo_test', ['FooTest.java']) target_roots = [] if create_some_resources: # Create a synthetic resource target. target_roots.append(self.make_target('some_resources', Resources)) target_roots.append(target) # Set the context with the two targets, one junit_tests target and # one synthetic resources target. # The synthetic resources target is to make sure we won't regress # in the future with bug like https://github.com/pantsbuild/pants/issues/508. Note # in that bug, the resources target must be the first one in the list. context = self.context(target_roots=target_roots) # Before we run the task, we need to inject the "runtime_classpath" with # the compiled test java classes that JUnitRun will know which test # classes to execute. In a normal run, this "runtime_classpath" will be # populated by java compilation step. self.populate_runtime_classpath(context=context, classpath=[test_classes_abs_path]) # Finally execute the task. self.execute(context)