Example #1
0
  def setUp(self):
    super(JvmToolTaskTestBase, self).setUp()

    # Use a synthetic subclass for proper isolation when bootstrapping within the test.
    bootstrap_scope = 'bootstrap_scope'
    self.bootstrap_task_type = self.synthesize_task_subtype(BootstrapJvmTools, bootstrap_scope)
    JvmToolMixin.reset_registered_tools()

    # Set some options:

    # 1. Cap BootstrapJvmTools memory usage in tests.  The Xmx was empirically arrived upon using
    #    -Xloggc and verifying no full gcs for a test using the full gamut of resolving a multi-jar
    #    tool, constructing a fat jar and then shading that fat jar.
    #
    # 2. Allow tests to read/write tool jars from the real artifact cache, so they don't
    #    each have to resolve and shade them every single time, which is a huge slowdown.
    #    Note that local artifact cache writes are atomic, so it's fine for multiple concurrent
    #    tests to write to it.
    #
    # Note that we don't have access to the invoking pants instance's options, so we assume that
    # its artifact cache is in the standard location.  If it isn't, worst case the tests will
    # populate a second cache at the standard location, which is no big deal.
    # TODO: We really need a straightforward way for pants's own tests to get to the enclosing
    # pants instance's options values.
    artifact_caches = [os.path.join(get_pants_cachedir(), 'artifact_cache')]
    self.set_options_for_scope(bootstrap_scope, jvm_options=['-Xmx128m'])
    self.set_options_for_scope('cache.{}'.format(bootstrap_scope),
                               read_from=artifact_caches,
                               write_to=artifact_caches)

    # Tool option defaults currently point to targets in the real BUILD.tools, so we copy it
    # into our test workspace.
    shutil.copy(os.path.join(self.real_build_root, 'BUILD.tools'), self.build_root)

    Bootstrapper.reset_instance()
Example #2
0
  def execute_junit_runner(self, content):

    # Create the temporary base test directory
    test_rel_path = 'tests/java/org/pantsbuild/foo'
    test_abs_path = os.path.join(self.build_root, test_rel_path)
    self.create_dir(test_rel_path)

    # Generate the temporary java test source code.
    test_java_file_rel_path = os.path.join(test_rel_path, 'FooTest.java')
    test_java_file_abs_path = os.path.join(self.build_root, test_java_file_rel_path)
    self.create_file(test_java_file_rel_path, content)

    # Invoke ivy to resolve classpath for junit.
    distribution = Distribution.cached(jdk=True)
    executor = SubprocessExecutor(distribution=distribution)
    classpath_file_abs_path = os.path.join(test_abs_path, 'junit.classpath')
    with subsystem_instance(IvySubsystem) as ivy_subsystem:
      ivy = Bootstrapper(ivy_subsystem=ivy_subsystem).ivy()
      ivy.execute(args=['-cachepath', classpath_file_abs_path,
                        '-dependency', 'junit', 'junit-dep', '4.10'], executor=executor)

    with open(classpath_file_abs_path) as fp:
      classpath = fp.read()

    # Now directly invoking javac to compile the test java code into java class
    # so later we can inject the class into products mapping for JUnitRun to execute
    # the test on.
    javac = distribution.binary('javac')
    subprocess.check_call(
      [javac, '-d', test_abs_path, '-cp', classpath, test_java_file_abs_path])

    # Create a java_tests target and a synthetic resource target.
    java_tests = self.create_library(test_rel_path, 'java_tests', 'foo_test', ['FooTest.java'])
    resources = self.make_target('some_resources', Resources)

    # Set the context with the two targets, one java_tests target and
    # one synthetic resources target.
    # The synthetic resources target is to make sure we won't regress
    # in the future with bug like https://github.com/pantsbuild/pants/issues/508. Note
    # in that bug, the resources target must be the first one in the list.
    context = self.context(target_roots=[resources, java_tests])

    # Before we run the task, we need to inject the "classes_by_target" with
    # the compiled test java classes that JUnitRun will know which test
    # classes to execute. In a normal run, this "classes_by_target" will be
    # populated by java compiling step.
    class_products = context.products.get_data(
      'classes_by_target', lambda: defaultdict(MultipleRootedProducts))
    java_tests_products = MultipleRootedProducts()
    java_tests_products.add_rel_paths(test_abs_path, ['FooTest.class'])
    class_products[java_tests] = java_tests_products

    # Also we need to add the FooTest.class's classpath to the compile_classpath
    # products data mapping so JUnitRun will be able to add that into the final
    # classpath under which the junit will be executed.
    self.populate_compile_classpath(context=context, classpath=[test_abs_path])

    # Finally execute the task.
    self.execute(context)
Example #3
0
  def execute_junit_runner(self, content):

    # Create the temporary base test directory
    test_rel_path = 'tests/java/org/pantsbuild/foo'
    test_abs_path = os.path.join(self.build_root, test_rel_path)
    self.create_dir(test_rel_path)

    # Generate the temporary java test source code.
    test_java_file_rel_path = os.path.join(test_rel_path, 'FooTest.java')
    test_java_file_abs_path = os.path.join(self.build_root, test_java_file_rel_path)
    self.create_file(test_java_file_rel_path, content)

    # Invoke ivy to resolve classpath for junit.
    classpath_file_abs_path = os.path.join(test_abs_path, 'junit.classpath')
    with subsystem_instance(IvySubsystem) as ivy_subsystem:
      distribution = DistributionLocator.cached(jdk=True)
      ivy = Bootstrapper(ivy_subsystem=ivy_subsystem).ivy()
      ivy.execute(args=['-cachepath', classpath_file_abs_path,
                        '-dependency', 'junit', 'junit-dep', '4.10'],
                  executor=SubprocessExecutor(distribution=distribution))

    with open(classpath_file_abs_path) as fp:
      classpath = fp.read()

    # Now directly invoking javac to compile the test java code into java class
    # so later we can inject the class into products mapping for JUnitRun to execute
    # the test on.
    javac = distribution.binary('javac')
    subprocess.check_call(
      [javac, '-d', test_abs_path, '-cp', classpath, test_java_file_abs_path])

    # Create a java_tests target and a synthetic resource target.
    java_tests = self.create_library(test_rel_path, 'java_tests', 'foo_test', ['FooTest.java'])
    resources = self.make_target('some_resources', Resources)

    # Set the context with the two targets, one java_tests target and
    # one synthetic resources target.
    # The synthetic resources target is to make sure we won't regress
    # in the future with bug like https://github.com/pantsbuild/pants/issues/508. Note
    # in that bug, the resources target must be the first one in the list.
    context = self.context(target_roots=[resources, java_tests])

    # Before we run the task, we need to inject the "classes_by_target" with
    # the compiled test java classes that JUnitRun will know which test
    # classes to execute. In a normal run, this "classes_by_target" will be
    # populated by java compiling step.
    class_products = context.products.get_data(
      'classes_by_target', lambda: defaultdict(MultipleRootedProducts))
    java_tests_products = MultipleRootedProducts()
    java_tests_products.add_rel_paths(test_abs_path, ['FooTest.class'])
    class_products[java_tests] = java_tests_products

    # Also we need to add the FooTest.class's classpath to the compile_classpath
    # products data mapping so JUnitRun will be able to add that into the final
    # classpath under which the junit will be executed.
    self.populate_compile_classpath(context=context, classpath=[test_abs_path])

    # Finally execute the task.
    self.execute(context)
Example #4
0
  def execute_junit_runner(self, content, create_some_resources=True, **kwargs):
    # Create the temporary base test directory
    test_rel_path = 'tests/java/org/pantsbuild/foo'
    test_abs_path = self.create_dir(test_rel_path)

    # Generate the temporary java test source code.
    test_java_file_rel_path = os.path.join(test_rel_path, 'FooTest.java')
    test_java_file_abs_path = self.create_file(test_java_file_rel_path, content)

    # Create the temporary classes directory under work dir
    test_classes_abs_path = self.create_workdir_dir(test_rel_path)

    # Invoke ivy to resolve classpath for junit.
    classpath_file_abs_path = os.path.join(test_abs_path, 'junit.classpath')
    ivy_subsystem = global_subsystem_instance(IvySubsystem)
    distribution = DistributionLocator.cached(jdk=True)
    ivy = Bootstrapper(ivy_subsystem=ivy_subsystem).ivy()
    ivy.execute(args=['-cachepath', classpath_file_abs_path,
                      '-dependency', 'junit', 'junit-dep', '4.10'],
                executor=SubprocessExecutor(distribution=distribution))

    with open(classpath_file_abs_path) as fp:
      classpath = fp.read()

    # Now directly invoking javac to compile the test java code into java class
    # so later we can inject the class into products mapping for JUnitRun to execute
    # the test on.
    javac = distribution.binary('javac')
    subprocess.check_call(
      [javac, '-d', test_classes_abs_path, '-cp', classpath, test_java_file_abs_path])

    # If a target_name is specified create a target with it, otherwise create a java_tests target.
    if 'target_name' in kwargs:
      target = self.target(kwargs['target_name'])
    else:
      target = self.create_library(test_rel_path, 'java_tests', 'foo_test', ['FooTest.java'])

    target_roots = []
    if create_some_resources:
      # Create a synthetic resource target.
      target_roots.append(self.make_target('some_resources', Resources))
    target_roots.append(target)

    # Set the context with the two targets, one java_tests target and
    # one synthetic resources target.
    # The synthetic resources target is to make sure we won't regress
    # in the future with bug like https://github.com/pantsbuild/pants/issues/508. Note
    # in that bug, the resources target must be the first one in the list.
    context = self.context(target_roots=target_roots)

    # Before we run the task, we need to inject the "runtime_classpath" with
    # the compiled test java classes that JUnitRun will know which test
    # classes to execute. In a normal run, this "runtime_classpath" will be
    # populated by java compilation step.
    self.populate_runtime_classpath(context=context, classpath=[test_classes_abs_path])

    # Finally execute the task.
    self.execute(context)
Example #5
0
 def test_fresh_bootstrap(self):
   with temporary_dir() as fresh_bootstrap_dir:
     self.set_bootstrap_options(pants_bootstrapdir=fresh_bootstrap_dir)
     # Initialize the Ivy subsystem
     self.context()
     Bootstrapper.default_ivy()
     bootstrap_jar_path = os.path.join(fresh_bootstrap_dir,
                                       'tools', 'jvm', 'ivy', 'bootstrap.jar')
     self.assertTrue(os.path.exists(bootstrap_jar_path))
Example #6
0
 def test_simple(self):
   ivy_subsystem = IvySubsystem.global_instance()
   bootstrapper = Bootstrapper(ivy_subsystem=ivy_subsystem)
   ivy = bootstrapper.ivy()
   self.assertIsNotNone(ivy.ivy_resolution_cache_dir)
   self.assertIsNone(ivy.ivy_settings)
   bootstrap_jar_path = os.path.join(ivy_subsystem.get_options().pants_bootstrapdir,
                                     'tools', 'jvm', 'ivy', 'bootstrap.jar')
   self.assertTrue(os.path.exists(bootstrap_jar_path))
Example #7
0
 def test_simple(self):
   with subsystem_instance(IvySubsystem) as ivy_subsystem:
     bootstrapper = Bootstrapper(ivy_subsystem=ivy_subsystem)
     ivy = bootstrapper.ivy()
     self.assertIsNotNone(ivy.ivy_cache_dir)
     self.assertIsNone(ivy.ivy_settings)
     bootstrap_jar_path = os.path.join(ivy_subsystem.get_options().pants_bootstrapdir,
                                       'tools', 'jvm', 'ivy', 'bootstrap.jar')
     self.assertTrue(os.path.exists(bootstrap_jar_path))
 def test_simple(self):
     ivy_subsystem = IvySubsystem.global_instance()
     bootstrapper = Bootstrapper(ivy_subsystem=ivy_subsystem)
     ivy = bootstrapper.ivy()
     self.assertIsNotNone(ivy.ivy_cache_dir)
     self.assertIsNone(ivy.ivy_settings)
     bootstrap_jar_path = os.path.join(
         ivy_subsystem.get_options().pants_bootstrapdir, "tools", "jvm", "ivy", "bootstrap.jar"
     )
     self.assertTrue(os.path.exists(bootstrap_jar_path))
Example #9
0
 def setUp(self):
   super(TaskTestBase, self).setUp()
   self._testing_task_type, self.options_scope = self.synthesize_task_subtype(self.task_type())
   # We locate the workdir below the pants_workdir, which BaseTest locates within
   # the BuildRoot.
   self._tmpdir = tempfile.mkdtemp(dir=self.pants_workdir)
   self._test_workdir = os.path.join(self._tmpdir, 'workdir')
   os.mkdir(self._test_workdir)
   Bootstrapper.reset_instance()
   IvySubsystem.reset_global_instance()
Example #10
0
  def test_parse_proxy_string(self):
    bootstrapper = Bootstrapper().instance()

    self.assertEquals(('example.com', 1234),
                      bootstrapper._parse_proxy_string('http://example.com:1234'))
    self.assertEquals(('secure-example.com', 999),
                      bootstrapper._parse_proxy_string('http://secure-example.com:999'))
    # trailing slash is ok
    self.assertEquals(('example.com', 1234),
                      bootstrapper._parse_proxy_string('http://example.com:1234/'))
Example #11
0
 def setUp(self):
   super(TaskTestBase, self).setUp()
   self._testing_task_type = self.synthesize_task_subtype(self.task_type(), self.options_scope)
   # We locate the workdir below the pants_workdir, which BaseTest locates within the BuildRoot.
   # BaseTest cleans this up, so we don't need to.  We give it a stable name, so that we can
   # use artifact caching to speed up tests.
   self._test_workdir = os.path.join(self.pants_workdir, self.task_type().stable_name())
   os.mkdir(self._test_workdir)
   # TODO: Push this down to JVM-related tests only? Seems wrong to have an ivy-specific
   # action in this non-JVM-specific, high-level base class.
   Bootstrapper.reset_instance()
Example #12
0
 def _temp_ivy_cache_dir(self):
   # We also reset the Bootstrapper since it hangs on to a ivy subsystem.
   old_instance = Bootstrapper._INSTANCE
   try:
     with temporary_dir() as ivy_cache_dir:
       Bootstrapper.reset_instance()
       # We set a temp ivy cache to ensure that the ivy we're using won't cache the temporary jar.
       self.set_options_for_scope('ivy', cache_dir=ivy_cache_dir)
       yield ivy_cache_dir
   finally:
     Bootstrapper._INSTANCE = old_instance
 def _temp_ivy_cache_dir(self):
     # We also reset the Bootstrapper since it hangs on to a ivy subsystem.
     old_instance = Bootstrapper._INSTANCE
     try:
         with temporary_dir() as ivy_cache_dir:
             Bootstrapper.reset_instance()
             # We set a temp ivy cache to ensure that the ivy we're using won't cache the temporary jar.
             self.set_options_for_scope('ivy', cache_dir=ivy_cache_dir)
             yield ivy_cache_dir
     finally:
         Bootstrapper._INSTANCE = old_instance
Example #14
0
 def setUp(self):
   super(TaskTestBase, self).setUp()
   self._testing_task_type, self.options_scope = self.synthesize_task_subtype(self.task_type())
   # We locate the workdir below the pants_workdir, which BaseTest locates within
   # the BuildRoot.
   self._tmpdir = tempfile.mkdtemp(dir=self.pants_workdir)
   self._test_workdir = os.path.join(self._tmpdir, 'workdir')
   os.mkdir(self._test_workdir)
   # TODO: Push this down to JVM-related tests only? Seems wrong to have an ivy-specific
   # action in this non-JVM-specific, high-level base class.
   Bootstrapper.reset_instance()
Example #15
0
 def setUp(self):
   super(TaskTestBase, self).setUp()
   self.options_scope = 'test_scope'
   self._testing_task_type = self.synthesize_task_subtype(self.task_type(), self.options_scope)
   # We locate the workdir below the pants_workdir, which BaseTest locates within the BuildRoot.
   # BaseTest cleans this up, so we don't need to.  We give it a stable name, so that we can
   # use artifact caching to speed up tests.
   self._test_workdir = os.path.join(self.pants_workdir, self.task_type().stable_name())
   os.mkdir(self._test_workdir)
   # TODO: Push this down to JVM-related tests only? Seems wrong to have an ivy-specific
   # action in this non-JVM-specific, high-level base class.
   Bootstrapper.reset_instance()
Example #16
0
    def execute_junit_runner(self, content):
        # Create the temporary base test directory
        test_rel_path = "tests/java/org/pantsbuild/foo"
        test_abs_path = self.create_dir(test_rel_path)

        # Generate the temporary java test source code.
        test_java_file_rel_path = os.path.join(test_rel_path, "FooTest.java")
        test_java_file_abs_path = self.create_file(test_java_file_rel_path, content)

        # Create the temporary classes directory under work dir
        test_classes_abs_path = self.create_workdir_dir(test_rel_path)

        # Invoke ivy to resolve classpath for junit.
        classpath_file_abs_path = os.path.join(test_abs_path, "junit.classpath")
        with subsystem_instance(IvySubsystem) as ivy_subsystem:
            distribution = DistributionLocator.cached(jdk=True)
            ivy = Bootstrapper(ivy_subsystem=ivy_subsystem).ivy()
            ivy.execute(
                args=["-cachepath", classpath_file_abs_path, "-dependency", "junit", "junit-dep", "4.10"],
                executor=SubprocessExecutor(distribution=distribution),
            )

        with open(classpath_file_abs_path) as fp:
            classpath = fp.read()

        # Now directly invoking javac to compile the test java code into java class
        # so later we can inject the class into products mapping for JUnitRun to execute
        # the test on.
        javac = distribution.binary("javac")
        subprocess.check_call([javac, "-d", test_classes_abs_path, "-cp", classpath, test_java_file_abs_path])

        # Create a java_tests target and a synthetic resource target.
        java_tests = self.create_library(test_rel_path, "java_tests", "foo_test", ["FooTest.java"])
        resources = self.make_target("some_resources", Resources)

        # Set the context with the two targets, one java_tests target and
        # one synthetic resources target.
        # The synthetic resources target is to make sure we won't regress
        # in the future with bug like https://github.com/pantsbuild/pants/issues/508. Note
        # in that bug, the resources target must be the first one in the list.
        context = self.context(target_roots=[resources, java_tests])

        # Before we run the task, we need to inject the "runtime_classpath" with
        # the compiled test java classes that JUnitRun will know which test
        # classes to execute. In a normal run, this "runtime_classpath" will be
        # populated by java compilation step.
        self.populate_runtime_classpath(context=context, classpath=[test_classes_abs_path])

        # Finally execute the task.
        self.execute(context)
Example #17
0
    def __init__(self, *args, **kwargs):
        super(IvyResolve, self).__init__(*args, **kwargs)

        self._ivy_bootstrapper = Bootstrapper.instance()
        self._cachedir = self._ivy_bootstrapper.ivy_cache_dir
        self._confs = self.context.config.getlist(self._CONFIG_SECTION,
                                                  'confs',
                                                  default=['default'])
        self._classpath_dir = os.path.join(self.workdir, 'mapped')

        self._outdir = self.get_options().outdir or os.path.join(
            self.workdir, 'reports')
        self._open = self.get_options().open
        self._report = self._open or self.get_options().report

        self._ivy_bootstrap_key = 'ivy'
        self.register_jvm_tool_from_config(self._ivy_bootstrap_key,
                                           self.context.config,
                                           ini_section=self._CONFIG_SECTION,
                                           ini_key='bootstrap-tools',
                                           default=['//:xalan'])

        self._ivy_utils = IvyUtils(config=self.context.config,
                                   log=self.context.log)

        # Typically this should be a local cache only, since classpaths aren't portable.
        self.setup_artifact_cache_from_config(
            config_section=self._CONFIG_SECTION)
Example #18
0
    def dumped_chroot(self, targets):
        python_repos = create_subsystem(PythonRepos)

        with subsystem_instance(IvySubsystem) as ivy_subsystem:
            ivy_bootstrapper = Bootstrapper(ivy_subsystem=ivy_subsystem)

            with subsystem_instance(
                    ThriftBinary.Factory) as thrift_binary_factory:
                interpreter_cache = PythonInterpreterCache(
                    self.python_setup, python_repos)
                interpreter_cache.setup()
                interpreters = list(
                    interpreter_cache.matches(
                        [self.python_setup.interpreter_requirement]))
                self.assertGreater(len(interpreters), 0)
                interpreter = interpreters[0]

                with temporary_dir() as chroot:
                    pex_builder = PEXBuilder(path=chroot,
                                             interpreter=interpreter)

                    python_chroot = PythonChroot(
                        python_setup=self.python_setup,
                        python_repos=python_repos,
                        ivy_bootstrapper=ivy_bootstrapper,
                        thrift_binary_factory=thrift_binary_factory.create,
                        interpreter=interpreter,
                        builder=pex_builder,
                        targets=targets,
                        platforms=['current'])
                    try:
                        python_chroot.dump()
                        yield pex_builder, python_chroot
                    finally:
                        python_chroot.delete()
Example #19
0
    def exec_ivy(cls, ivy, confs, ivyxml, args, jvm_options, executor,
                 workunit_name, workunit_factory):
        ivy = ivy or Bootstrapper.default_ivy()

        ivy_args = ['-ivy', ivyxml]
        ivy_args.append('-confs')
        ivy_args.extend(confs)
        ivy_args.extend(args)

        ivy_jvm_options = list(jvm_options)
        # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng.
        ivy_jvm_options.append('-Dsun.io.useCanonCaches=false')

        runner = ivy.runner(jvm_options=ivy_jvm_options,
                            args=ivy_args,
                            executor=executor)
        try:
            result = execute_runner(runner,
                                    workunit_factory=workunit_factory,
                                    workunit_name=workunit_name)
            if result != 0:
                raise IvyUtils.IvyError(
                    'Ivy returned {result}. cmd={cmd}'.format(result=result,
                                                              cmd=runner.cmd))
        except runner.executor.Error as e:
            raise IvyUtils.IvyError(e)
Example #20
0
  def publish(self, ivyxml_path, jar, entry, repo, published):
    """Run ivy to publish a jar.  ivyxml_path is the path to the ivy file; published
    is a list of jars published so far (including this one). entry is a pushdb entry."""
    jvm_args = self._ivy_jvm_args(repo)
    resolver = repo['resolver']
    path = repo.get('path')

    try:
      ivy = Bootstrapper.default_ivy()
    except Bootstrapper.Error as e:
      raise TaskError('Failed to push {0}! {1}'.format(pushdb_coordinate(jar, entry), e))

    ivysettings = self.generate_ivysettings(ivy, published, publish_local=path)
    args = [
      '-settings', ivysettings,
      '-ivy', ivyxml_path,
      '-deliverto', '%s/[organisation]/[module]/ivy-[revision].xml' % self.workdir,
      '-publish', resolver,
      '-publishpattern', '%s/[organisation]/[module]/'
                         '[artifact]-[revision](-[classifier]).[ext]' % self.workdir,
      '-revision', entry.version().version(),
      '-m2compatible',
    ]

    if LogOptions.stderr_log_level() == logging.DEBUG:
      args.append('-verbose')

    if self.local_snapshot:
      args.append('-overwrite')

    try:
      ivy.execute(jvm_options=jvm_args, args=args,
                  workunit_factory=self.context.new_workunit, workunit_name='jar-publish')
    except Ivy.Error as e:
      raise TaskError('Failed to push {0}! {1}'.format(pushdb_coordinate(jar, entry), e))
Example #21
0
          def publish(ivyxml_path):
            ivysettings = self.generate_ivysettings(published, publish_local=path)
            args = [
              '-settings', ivysettings,
              '-ivy', ivyxml_path,
              '-deliverto', '%s/[organisation]/[module]/ivy-[revision].xml' % self.workdir,
              '-publish', resolver,
              '-publishpattern', '%s/[organisation]/[module]/'
                                 '[artifact]-[revision](-[classifier]).[ext]' % self.workdir,
              '-revision', newver.version(),
              '-m2compatible',
            ]

            if LogOptions.stderr_log_level() == logging.DEBUG:
              args.append('-verbose')

            if self.snapshot:
              args.append('-overwrite')

            try:
              ivy = Bootstrapper.default_ivy()
              ivy.execute(jvm_options=jvm_args, args=args,
                          workunit_factory=self.context.new_workunit, workunit_name='jar-publish')
            except (Bootstrapper.Error, Ivy.Error) as e:
              raise TaskError('Failed to push %s! %s' % (jar_coordinate(jar, newver.version()), e))
Example #22
0
  def exec_ivy(cls, ivy, confs, ivyxml, args,
               jvm_options,
               executor,
               workunit_name,
               workunit_factory):
    """
    :API: public
    """
    ivy = ivy or Bootstrapper.default_ivy()

    ivy_args = ['-ivy', ivyxml]
    ivy_args.append('-confs')
    ivy_args.extend(confs)
    ivy_args.extend(args)

    ivy_jvm_options = list(jvm_options)
    # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng.
    ivy_jvm_options.append('-Dsun.io.useCanonCaches=false')

    runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor)
    try:
      with ivy.resolution_lock:
        result = execute_runner(runner, workunit_factory=workunit_factory,
                                workunit_name=workunit_name)
      if result != 0:
        raise IvyUtils.IvyError('Ivy returned {result}. cmd={cmd}'.format(result=result, cmd=runner.cmd))
    except runner.executor.Error as e:
      raise IvyUtils.IvyError(e)
  def dumped_chroot(self, targets):
    # TODO(benjy): We shouldn't need to mention DistributionLocator here, as IvySubsystem
    # declares it as a dependency. However if we don't then test_antlr() below fails on
    # uninitialized options for that subsystem.  Hopefully my pending (as of 9/2016) change
    # to clean up how we initialize and create instances of subsystems in tests will make
    # this problem go away.
    self.context(for_subsystems=[PythonRepos, PythonSetup, IvySubsystem,
                                 DistributionLocator, ThriftBinary.Factory, BinaryUtil.Factory])
    python_repos = PythonRepos.global_instance()
    ivy_bootstrapper = Bootstrapper(ivy_subsystem=IvySubsystem.global_instance())
    thrift_binary_factory = ThriftBinary.Factory.global_instance().create

    interpreter_cache = PythonInterpreterCache(self.python_setup, python_repos)
    interpreter = interpreter_cache.select_interpreter_for_targets(targets)
    self.assertIsNotNone(interpreter)

    with temporary_dir() as chroot:
      pex_builder = PEXBuilder(path=chroot, interpreter=interpreter)

      python_chroot = PythonChroot(python_setup=self.python_setup,
                                   python_repos=python_repos,
                                   ivy_bootstrapper=ivy_bootstrapper,
                                   thrift_binary_factory=thrift_binary_factory,
                                   interpreter=interpreter,
                                   builder=pex_builder,
                                   targets=targets,
                                   platforms=['current'])
      try:
        python_chroot.dump()
        yield pex_builder, python_chroot
      finally:
        python_chroot.delete()
Example #24
0
    def __init__(self, context, workdir, confs=None):
        super(IvyResolve, self).__init__(context, workdir)

        self._ivy_bootstrapper = Bootstrapper.instance()
        self._cachedir = self._ivy_bootstrapper.ivy_cache_dir
        self._confs = confs or context.config.getlist(
            'ivy-resolve', 'confs', default=['default'])
        self._classpath_dir = os.path.join(self.workdir, 'mapped')

        self._outdir = context.options.ivy_resolve_outdir or os.path.join(
            self.workdir, 'reports')
        self._open = context.options.ivy_resolve_open
        self._report = self._open or context.options.ivy_resolve_report

        self._ivy_bootstrap_key = 'ivy'
        ivy_bootstrap_tools = context.config.getlist('ivy-resolve',
                                                     'bootstrap-tools',
                                                     ':xalan')
        self.register_jvm_tool(self._ivy_bootstrap_key, ivy_bootstrap_tools)

        self._ivy_utils = IvyUtils(config=context.config,
                                   options=context.options,
                                   log=context.log)
        context.products.require_data('exclusives_groups')

        # Typically this should be a local cache only, since classpaths aren't portable.
        self.setup_artifact_cache_from_config(config_section='ivy-resolve')
Example #25
0
          def publish(ivyxml_path):
            try:
              ivy = Bootstrapper.default_ivy()
            except Bootstrapper.Error as e:
              raise TaskError('Failed to push %s! %s' % (jar_coordinate(jar, newver.version()), e))

            ivysettings = self.generate_ivysettings(ivy, published, publish_local=path)
            args = [
              '-settings', ivysettings,
              '-ivy', ivyxml_path,
              '-deliverto', '%s/[organisation]/[module]/ivy-[revision].xml' % self.workdir,
              '-publish', resolver,
              '-publishpattern', '%s/[organisation]/[module]/'
                                 '[artifact]-[revision](-[classifier]).[ext]' % self.workdir,
              '-revision', newver.version(),
              '-m2compatible',
            ]

            if LogOptions.stderr_log_level() == logging.DEBUG:
              args.append('-verbose')

            if self.snapshot:
              args.append('-overwrite')

            try:
              ivy.execute(jvm_options=jvm_args, args=args,
                          workunit_factory=self.context.new_workunit, workunit_name='jar-publish')
            except Ivy.Error as e:
              raise TaskError('Failed to push %s! %s' % (jar_coordinate(jar, newver.version()), e))
Example #26
0
    def setUp(self):
        """
        :API: public
        """
        super().setUp()

        # Use a synthetic subclass for proper isolation when bootstrapping within the test.
        bootstrap_scope = "bootstrap_scope"
        self.bootstrap_task_type = self.synthesize_task_subtype(
            BootstrapJvmTools, bootstrap_scope)
        JvmToolMixin.reset_registered_tools()

        # Set some options:

        # 1. Cap BootstrapJvmTools memory usage in tests.  The Xmx was empirically arrived upon using
        #    -Xloggc and verifying no full gcs for a test using the full gamut of resolving a multi-jar
        #    tool, constructing a fat jar and then shading that fat jar.
        #
        # 2. Allow tests to read/write tool jars from the real artifact cache, so they don't
        #    each have to resolve and shade them every single time, which is a huge slowdown.
        #    Note that local artifact cache writes are atomic, so it's fine for multiple concurrent
        #    tests to write to it.
        #
        # Note that we don't have access to the invoking pants instance's options, so we assume that
        # its artifact cache is in the standard location.  If it isn't, worst case the tests will
        # populate a second cache at the standard location, which is no big deal.
        # TODO: We really need a straightforward way for pants's own tests to get to the enclosing
        # pants instance's options values.
        artifact_caches = [
            os.path.join(get_pants_cachedir(), "artifact_cache")
        ]
        self.set_options_for_scope(
            bootstrap_scope,
            execution_strategy=NailgunTask.ExecutionStrategy.subprocess,
            jvm_options=["-Xmx128m"],
        )
        self.set_options_for_scope(f"cache.{bootstrap_scope}",
                                   read_from=artifact_caches,
                                   write_to=artifact_caches)

        # Copy into synthetic build-root
        shutil.copy("BUILD.tools", self.build_root)
        build_root_third_party = os.path.join(self.build_root, "3rdparty")
        safe_mkdir(build_root_third_party)
        shutil.copy(os.path.join("3rdparty", "BUILD"), build_root_third_party)

        Bootstrapper.reset_instance()
Example #27
0
  def exec_ivy(self,
               target_workdir,
               targets,
               args,
               confs=None,
               ivy=None,
               workunit_name='ivy',
               workunit_factory=None,
               symlink_ivyxml=False,
               jars=None):

    ivy = ivy or Bootstrapper.default_ivy()
    if not isinstance(ivy, Ivy):
      raise ValueError('The ivy argument supplied must be an Ivy instance, given %s of type %s'
                       % (ivy, type(ivy)))

    ivyxml = os.path.join(target_workdir, 'ivy.xml')

    if not jars:
      jars, excludes = self._calculate_classpath(targets)
    else:
      excludes = set()

    ivy_args = ['-ivy', ivyxml]

    confs_to_resolve = confs or ['default']
    ivy_args.append('-confs')
    ivy_args.extend(confs_to_resolve)

    ivy_args.extend(args)
    if not self._transitive:
      ivy_args.append('-notransitive')
    ivy_args.extend(self._args)

    def safe_link(src, dest):
      try:
        os.unlink(dest)
      except OSError as e:
        if e.errno != errno.ENOENT:
          raise
      os.symlink(src, dest)

    with IvyUtils.ivy_lock:
      self._generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve)
      runner = ivy.runner(jvm_options=self._jvm_options, args=ivy_args)
      try:
        result = util.execute_runner(runner,
                                     workunit_factory=workunit_factory,
                                     workunit_name=workunit_name)

        # Symlink to the current ivy.xml file (useful for IDEs that read it).
        if symlink_ivyxml:
          ivyxml_symlink = os.path.join(self._workdir, 'ivy.xml')
          safe_link(ivyxml, ivyxml_symlink)

        if result != 0:
          raise TaskError('Ivy returned %d' % result)
      except runner.executor.Error as e:
        raise TaskError(e)
Example #28
0
 def test_depmap_jar_path(self):
   with temporary_dir(root_dir=self.workdir_root()) as workdir:
     test_target = 'examples/tests/java/com/pants/examples/usethrift:usethrift'
     json_data = self.run_depmap_project_info(test_target, workdir)
     ivy_cache_dir = Bootstrapper.instance().ivy_cache_dir
     self.assertEquals(json_data.get('libraries').get('commons-lang:commons-lang:2.5'),
                     [os.path.join(ivy_cache_dir,
                                   'commons-lang/commons-lang/jars/commons-lang-2.5.jar')])
Example #29
0
 def test_simple(self):
   bootstrapper = Bootstrapper.instance()
   ivy = bootstrapper.ivy()
   self.assertIsNotNone(ivy.ivy_cache_dir)
   self.assertIsNone(ivy.ivy_settings)
   bootstrap_jar_path = os.path.join(self.test_workdir,
                                     'tools', 'jvm', 'ivy', 'bootstrap.jar')
   self.assertTrue(os.path.exists(bootstrap_jar_path))
Example #30
0
 def test_simple(self):
     bootstrapper = Bootstrapper.instance()
     ivy = bootstrapper.ivy()
     self.assertIsNotNone(ivy.ivy_cache_dir)
     self.assertIsNone(ivy.ivy_settings)
     bootstrap_jar_path = os.path.join(self.test_workdir, 'tools', 'jvm',
                                       'ivy', 'bootstrap.jar')
     self.assertTrue(os.path.exists(bootstrap_jar_path))
Example #31
0
    def publish(self, publications, jar, entry, repo, published):
        """Run ivy to publish a jar.  ivyxml_path is the path to the ivy file; published
    is a list of jars published so far (including this one). entry is a pushdb entry."""

        try:
            ivy = Bootstrapper.default_ivy()
        except Bootstrapper.Error as e:
            raise TaskError('Failed to push {0}! {1}'.format(
                pushdb_coordinate(jar, entry), e))

        path = repo.get('path')
        ivysettings = self.generate_ivysettings(ivy,
                                                published,
                                                publish_local=path)

        version = entry.version().version()
        ivyxml = self.generate_ivy(jar, version, publications)

        resolver = repo['resolver']
        args = [
            '-settings',
            ivysettings,
            '-ivy',
            ivyxml,

            # Without this setting, the ivy.xml is delivered to the CWD, littering the workspace.  We
            # don't need the ivy.xml, so just give it path under the workdir we won't use.
            '-deliverto',
            ivyxml + '.unused',
            '-publish',
            resolver,
            '-publishpattern',
            '{}/[organisation]/[module]/'
            '[artifact]-[revision](-[classifier]).[ext]'.format(self.workdir),
            '-revision',
            version,
            '-m2compatible',
        ]

        # TODO(John Sirois): global logging options should be hidden behind some sort of log manager
        # that we can:
        # a.) obtain a handle to (dependency injection or manual plumbing)
        # b.) query for log detail, ie: `if log_manager.is_verbose:`
        if self.get_options().level == 'debug':
            args.append('-verbose')

        if self.local_snapshot:
            args.append('-overwrite')

        try:
            jvm_options = self._ivy_jvm_options(repo)
            ivy.execute(jvm_options=jvm_options,
                        args=args,
                        workunit_factory=self.context.new_workunit,
                        workunit_name='ivy-publish')
        except Ivy.Error as e:
            raise TaskError('Failed to push {0}! {1}'.format(
                pushdb_coordinate(jar, entry), e))
  def setUp(self):
    super(JvmToolTaskTestBase, self).setUp()

    # Use a synthetic subclass for proper isolation when bootstrapping within the test.
    bootstrap_scope = 'bootstrap_scope'
    self.bootstrap_task_type = self.synthesize_task_subtype(BootstrapJvmTools, bootstrap_scope)
    JvmToolMixin.reset_registered_tools()

    # Cap BootstrapJvmTools memory usage in tests.  The Xmx was empirically arrived upon using
    # -Xloggc and verifying no full gcs for a test using the full gamut of resolving a multi-jar
    # tool, constructing a fat jar and then shading that fat jar.
    self.set_options_for_scope(bootstrap_scope, jvm_options=['-Xmx128m'])

    # Tool option defaults currently point to targets in the real BUILD.tools, so we copy it
    # into our test workspace.
    shutil.copy(os.path.join(self.real_build_root, 'BUILD.tools'), self.build_root)

    Bootstrapper.reset_instance()
 def test_depmap_jar_path(self):
   with temporary_dir(root_dir=self.workdir_root()) as workdir:
     test_target = 'examples/tests/java/com/pants/examples/usethrift:usethrift'
     json_data = self.run_depmap_project_info(test_target, workdir)
     # Hack because Bootstrapper.instance() reads config from cache. Will go away after we plumb
     # options into IvyUtil properly.
     Config.cache(Config.load())
     ivy_cache_dir = Bootstrapper.instance().ivy_cache_dir
     self.assertEquals(json_data.get('libraries').get('commons-lang:commons-lang:2.5'),
                     [os.path.join(ivy_cache_dir,
                                   'commons-lang/commons-lang/jars/commons-lang-2.5.jar')])
Example #34
0
    def publish(self, publications, jar, entry, repo, published):
        """Run ivy to publish a jar.  ivyxml_path is the path to the ivy file; published
    is a list of jars published so far (including this one). entry is a pushdb entry."""

        try:
            ivy = Bootstrapper.default_ivy()
        except Bootstrapper.Error as e:
            raise TaskError("Failed to push {0}! {1}".format(pushdb_coordinate(jar, entry), e))

        path = repo.get("path")
        ivysettings = self.generate_ivysettings(ivy, published, publish_local=path)

        version = entry.version().version()
        ivyxml = self.generate_ivy(jar, version, publications)

        resolver = repo["resolver"]
        args = [
            "-settings",
            ivysettings,
            "-ivy",
            ivyxml,
            # Without this setting, the ivy.xml is delivered to the CWD, littering the workspace.  We
            # don't need the ivy.xml, so just give it path under the workdir we won't use.
            "-deliverto",
            ivyxml + ".unused",
            "-publish",
            resolver,
            "-publishpattern",
            "{}/[organisation]/[module]/" "[artifact]-[revision](-[classifier]).[ext]".format(self.workdir),
            "-revision",
            version,
            "-m2compatible",
        ]

        # TODO(John Sirois): global logging options should be hidden behind some sort of log manager
        # that we can:
        # a.) obtain a handle to (dependency injection or manual plumbing)
        # b.) query for log detail, ie: `if log_manager.is_verbose:`
        if self.get_options().level == "debug":
            args.append("-verbose")

        if self.local_snapshot:
            args.append("-overwrite")

        try:
            jvm_options = self._ivy_jvm_options(repo)
            ivy.execute(
                jvm_options=jvm_options,
                args=args,
                workunit_factory=self.context.new_workunit,
                workunit_name="ivy-publish",
            )
        except Ivy.Error as e:
            raise TaskError("Failed to push {0}! {1}".format(pushdb_coordinate(jar, entry), e))
Example #35
0
    def mapjars(self,
                genmap,
                target,
                executor,
                workunit_factory=None,
                jars=None):
        """Resolves jars for the target and stores their locations in genmap.
    :param genmap: The jar_dependencies ProductMapping entry for the required products.
    :param target: The target whose jar dependencies are being retrieved.
    :param jars: If specified, resolves the given jars rather than
    :type jars: List of :class:`pants.backend.jvm.targets.jar_dependency.JarDependency` (jar())
      objects.
    """
        mapdir = os.path.join(self.mapto_dir(), target.id)
        safe_mkdir(mapdir, clean=True)
        ivyargs = [
            '-retrieve',
            '%s/[organisation]/[artifact]/[conf]/'
            '[organisation]-[artifact]-[revision](-[classifier]).[ext]' %
            mapdir,
            '-symlink',
        ]
        self.exec_ivy(mapdir, [target],
                      ivyargs,
                      confs=target.payload.configurations,
                      ivy=Bootstrapper.default_ivy(executor),
                      workunit_factory=workunit_factory,
                      workunit_name='map-jars',
                      jars=jars)

        for org in os.listdir(mapdir):
            orgdir = os.path.join(mapdir, org)
            if os.path.isdir(orgdir):
                for name in os.listdir(orgdir):
                    artifactdir = os.path.join(orgdir, name)
                    if os.path.isdir(artifactdir):
                        for conf in os.listdir(artifactdir):
                            confdir = os.path.join(artifactdir, conf)
                            for f in os.listdir(confdir):
                                if self.is_mappable_artifact(org, name, f):
                                    # TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a
                                    # conf whitelist
                                    genmap.add(org, confdir).append(f)
                                    genmap.add((org, name), confdir).append(f)

                                    genmap.add(target, confdir).append(f)
                                    genmap.add((target, conf),
                                               confdir).append(f)
                                    genmap.add((org, name, conf),
                                               confdir).append(f)
Example #36
0
  def publish(self, publications, jar, version, repo, published):
    """Run ivy to publish a jar.

    :param str ivyxml_path: The path to the ivy file.
    :param list published: A list of jars published so far (including this one).
    """

    try:
      ivy = Bootstrapper.default_ivy()
    except Bootstrapper.Error as e:
      raise TaskError('Failed to push {0}! {1}'.format(jar, e))

    if ivy.ivy_settings is None:
      raise TaskError('An ivysettings.xml with writeable resolvers is required for publishing, '
                      'but none was configured.')

    path = repo.get('path')

    ivysettings = self.generate_ivysettings(self.fetch_ivysettings(ivy), published, publish_local=path)
    ivyxml = self.generate_ivy(jar, version, publications)
    resolver = repo['resolver']
    args = [
      '-settings', ivysettings,
      '-ivy', ivyxml,

      # Without this setting, the ivy.xml is delivered to the CWD, littering the workspace.  We
      # don't need the ivy.xml, so just give it path under the workdir we won't use.
      '-deliverto', ivyxml + '.unused',

      '-publish', resolver,
      '-publishpattern', '{}/[organisation]/[module]/'
                         '[artifact]-[revision](-[classifier]).[ext]'.format(self.workdir),
      '-revision', version,
      '-m2compatible',
    ]

    if self.get_options().level == 'debug':
      args.append('-verbose')

    if self.get_options().local:
      args.append('-overwrite')

    if not self.get_options().dryrun:
      try:
        ivy.execute(jvm_options=self._ivy_jvm_options(repo), args=args,
                    workunit_factory=self.context.new_workunit, workunit_name='ivy-publish')
      except Ivy.Error as e:
        raise TaskError('Failed to push {0}! {1}'.format(jar, e))
    else:
      print("\nDRYRUN- would have pushed:{} using {} resolver.".format(self.jar_coordinate(jar, version), resolver))
Example #37
0
    def _exec_ivy(self,
                  target_workdir,
                  targets,
                  args,
                  executor=None,
                  confs=None,
                  ivy=None,
                  workunit_name='ivy',
                  use_soft_excludes=False,
                  resolve_hash_name=None):
        ivy_jvm_options = self.get_options().jvm_options[:]
        # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng.
        ivy_jvm_options.append('-Dsun.io.useCanonCaches=false')

        ivy = ivy or Bootstrapper.default_ivy()
        ivyxml = os.path.join(target_workdir, 'ivy.xml')

        ivy_args = ['-ivy', ivyxml]

        confs_to_resolve = confs or ('default', )
        ivy_args.append('-confs')
        ivy_args.extend(confs_to_resolve)
        ivy_args.extend(args)

        # TODO(John Sirois): merge the code below into IvyUtils or up here; either way, better
        # diagnostics can be had in `IvyUtils.generate_ivy` if this is done.
        # See: https://github.com/pantsbuild/pants/issues/2239
        try:
            jars, excludes = IvyUtils.calculate_classpath(
                targets, gather_excludes=not use_soft_excludes)
            with IvyUtils.ivy_lock:
                IvyUtils.generate_ivy(targets, jars, excludes, ivyxml,
                                      confs_to_resolve, resolve_hash_name)
                runner = ivy.runner(jvm_options=ivy_jvm_options,
                                    args=ivy_args,
                                    executor=executor)
                try:
                    result = execute_runner(
                        runner,
                        workunit_factory=self.context.new_workunit,
                        workunit_name=workunit_name)
                    if result != 0:
                        raise self.Error(
                            'Ivy returned {result}. cmd={cmd}'.format(
                                result=result, cmd=runner.cmd))
                except runner.executor.Error as e:
                    raise self.Error(e)
        except IvyUtils.IvyError as e:
            raise self.Error('Failed to prepare ivy resolve: {}'.format(e))
Example #38
0
 def test_depmap_jar_path(self):
     with temporary_dir(root_dir=self.workdir_root()) as workdir:
         test_target = 'examples/tests/java/com/pants/examples/usethrift:usethrift'
         json_data = self.run_depmap_project_info(test_target, workdir)
         # Hack because Bootstrapper.instance() reads config from cache. Will go away after we plumb
         # options into IvyUtil properly.
         Config.cache(Config.load())
         ivy_cache_dir = Bootstrapper.instance().ivy_cache_dir
         self.assertEquals(
             json_data.get('libraries').get(
                 'commons-lang:commons-lang:2.5'),
             [
                 os.path.join(
                     ivy_cache_dir,
                     'commons-lang/commons-lang/jars/commons-lang-2.5.jar')
             ])
Example #39
0
  def __init__(self, *args, **kwargs):
    super(IvyResolve, self).__init__(*args, **kwargs)

    self._ivy_bootstrapper = Bootstrapper.instance()
    self._cachedir = self._ivy_bootstrapper.ivy_cache_dir
    self._confs = self.context.config.getlist(self._CONFIG_SECTION, 'confs', default=['default'])
    self._classpath_dir = os.path.join(self.workdir, 'mapped')

    self._outdir = self.get_options().outdir or os.path.join(self.workdir, 'reports')
    self._open = self.get_options().open
    self._report = self._open or self.get_options().report

    self._ivy_utils = IvyUtils(config=self.context.config, log=self.context.log)

    # Typically this should be a local cache only, since classpaths aren't portable.
    self.setup_artifact_cache_from_config(config_section=self._CONFIG_SECTION)
Example #40
0
    def exec_ivy(self,
                 target_workdir,
                 targets,
                 args,
                 executor=None,
                 confs=None,
                 ivy=None,
                 workunit_name='ivy',
                 jars=None,
                 use_soft_excludes=False,
                 resolve_hash_name=None):
        ivy_jvm_options = copy.copy(self.get_options().jvm_options)
        # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng.
        ivy_jvm_options.append('-Dsun.io.useCanonCaches=false')

        ivy = ivy or Bootstrapper.default_ivy()
        ivyxml = os.path.join(target_workdir, 'ivy.xml')

        if not jars:
            jars, excludes = IvyUtils.calculate_classpath(
                targets, gather_excludes=not use_soft_excludes)
        else:
            excludes = set()

        ivy_args = ['-ivy', ivyxml]

        confs_to_resolve = confs or ['default']
        ivy_args.append('-confs')
        ivy_args.extend(confs_to_resolve)
        ivy_args.extend(args)

        with IvyUtils.ivy_lock:
            IvyUtils.generate_ivy(targets, jars, excludes, ivyxml,
                                  confs_to_resolve, resolve_hash_name)
            runner = ivy.runner(jvm_options=ivy_jvm_options,
                                args=ivy_args,
                                executor=executor)
            try:
                result = execute_runner(
                    runner,
                    workunit_factory=self.context.new_workunit,
                    workunit_name=workunit_name)
                if result != 0:
                    raise TaskError('Ivy returned {result}. cmd={cmd}'.format(
                        result=result, cmd=runner.cmd))
            except runner.executor.Error as e:
                raise TaskError(e)
Example #41
0
  def __init__(self, *args, **kwargs):
    super(IvyResolve, self).__init__(*args, **kwargs)

    self._ivy_bootstrapper = Bootstrapper.instance()
    self._cachedir = self._ivy_bootstrapper.ivy_cache_dir
    self._classpath_dir = os.path.join(self.workdir, 'mapped')
    self._outdir = self.get_options().outdir or os.path.join(self.workdir, 'reports')
    self._open = self.get_options().open
    self._report = self._open or self.get_options().report
    self._confs = None

    self._args = []
    for arg in self.get_options().args:
      self._args.extend(safe_shlex_split(arg))

    # Typically this should be a local cache only, since classpaths aren't portable.
    self.setup_artifact_cache()
Example #42
0
    def run_antlrs(self, output_dir):
        # TODO(John Sirois): graduate to a JvmToolTask and/or merge with the java code gen AntlrGen
        # task.
        args = [
            '-dependency', 'org.antlr', 'antlr', self.target.antlr_version,
            '-types', 'jar', '-main', 'org.antlr.Tool', '--', '-fo', output_dir
        ]
        for source in self.target.sources_relative_to_buildroot():
            abs_path = os.path.join(get_buildroot(), source)
            args.append(abs_path)

        try:
            ivy = Bootstrapper.default_ivy()
            ivy.execute(
                args=args
            )  # TODO: Needs a workunit, when we have a context here.
        except (Bootstrapper.Error, Ivy.Error) as e:
            raise TaskError('ANTLR generation failed! {0}'.format(e))
Example #43
0
  def run_antlrs(self, output_dir):
    # TODO(John Sirois): graduate to a JvmToolTask and/or merge with the java code gen AntlrGen
    # task.
    args = [
      '-dependency', 'org.antlr', 'antlr', self.target.antlr_version,
      '-types', 'jar',
      '-main', 'org.antlr.Tool',
      '--', '-fo', output_dir
    ]
    for source in self.target.sources_relative_to_buildroot():
      abs_path = os.path.join(get_buildroot(), source)
      args.append(abs_path)

    try:
      ivy = Bootstrapper.default_ivy()
      ivy.execute(args=args)  # TODO: Needs a workunit, when we have a context here.
    except (Bootstrapper.Error, Ivy.Error) as e:
      raise TaskError('ANTLR generation failed! {0}'.format(e))
Example #44
0
    def run_antlrs(self, output_dir):
        args = [
            '-dependency', 'org.antlr', 'antlr', self.target.antlr_version,
            '-types', 'jar', '-main', 'org.antlr.Tool', '--', '-fo', output_dir
        ]
        for source in self.target.sources_relative_to_buildroot():
            abs_path = os.path.join(get_buildroot(), source)
            args.append(abs_path)

        try:
            ivy = Bootstrapper.default_ivy()
            ivy.execute(
                args=args
            )  # TODO: Needs a workunit, when we have a context here.
            return True
        except (Bootstrapper.Error, Ivy.Error) as e:
            print('ANTLR generation failed! %s' % e, file=sys.stderr)
            return False
Example #45
0
    def __init__(self, *args, **kwargs):
        super(IvyResolve, self).__init__(*args, **kwargs)

        self._ivy_bootstrapper = Bootstrapper.instance()
        self._cachedir = self._ivy_bootstrapper.ivy_cache_dir
        self._classpath_dir = os.path.join(self.workdir, 'mapped')
        self._outdir = self.get_options().outdir or os.path.join(
            self.workdir, 'reports')
        self._open = self.get_options().open
        self._report = self._open or self.get_options().report
        self._confs = None

        self._args = []
        for arg in self.get_options().args:
            self._args.extend(safe_shlex_split(arg))

        # Typically this should be a local cache only, since classpaths aren't portable.
        self.setup_artifact_cache()
Example #46
0
    def _exec_ivy(
        self,
        target_workdir,
        targets,
        args,
        executor=None,
        confs=None,
        ivy=None,
        workunit_name="ivy",
        use_soft_excludes=False,
        resolve_hash_name=None,
    ):
        ivy_jvm_options = self.get_options().jvm_options[:]
        # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng.
        ivy_jvm_options.append("-Dsun.io.useCanonCaches=false")

        ivy = ivy or Bootstrapper.default_ivy()
        ivyxml = os.path.join(target_workdir, "ivy.xml")

        ivy_args = ["-ivy", ivyxml]

        confs_to_resolve = confs or ("default",)
        ivy_args.append("-confs")
        ivy_args.extend(confs_to_resolve)
        ivy_args.extend(args)

        # TODO(John Sirois): merge the code below into IvyUtils or up here; either way, better
        # diagnostics can be had in `IvyUtils.generate_ivy` if this is done.
        # See: https://github.com/pantsbuild/pants/issues/2239
        try:
            jars, excludes = IvyUtils.calculate_classpath(targets, gather_excludes=not use_soft_excludes)
            with IvyUtils.ivy_lock:
                IvyUtils.generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve, resolve_hash_name)
                runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor)
                try:
                    result = execute_runner(
                        runner, workunit_factory=self.context.new_workunit, workunit_name=workunit_name
                    )
                    if result != 0:
                        raise self.Error("Ivy returned {result}. cmd={cmd}".format(result=result, cmd=runner.cmd))
                except runner.executor.Error as e:
                    raise self.Error(e)
        except IvyUtils.IvyError as e:
            raise self.Error("Failed to prepare ivy resolve: {}".format(e))
Example #47
0
  def mapjars(self, genmap, target, executor, jars=None):
    """Resolves jars for the target and stores their locations in genmap.

    :param genmap: The jar_dependencies ProductMapping entry for the required products.
    :param target: The target whose jar dependencies are being retrieved.
    :param jars: If specified, resolves the given jars rather than
    :type jars: List of :class:`pants.backend.jvm.targets.jar_dependency.JarDependency` (jar())
      objects.
    """
    mapdir = os.path.join(self.workdir, 'mapped-jars', target.id)
    safe_mkdir(mapdir, clean=True)
    ivyargs = [
      '-retrieve', '%s/[organisation]/[artifact]/[conf]/'
                   '[organisation]-[artifact]-[revision](-[classifier]).[ext]' % mapdir,
      '-symlink',
    ]
    confs = maybe_list(target.payload.get_field_value('configurations') or [])
    self.exec_ivy(mapdir,
                  [target],
                  executor=executor,
                  args=ivyargs,
                  confs=confs,
                  ivy=Bootstrapper.default_ivy(),
                  workunit_name='map-jars',
                  jars=jars)

    for org in os.listdir(mapdir):
      orgdir = os.path.join(mapdir, org)
      if os.path.isdir(orgdir):
        for name in os.listdir(orgdir):
          artifactdir = os.path.join(orgdir, name)
          if os.path.isdir(artifactdir):
            for conf in os.listdir(artifactdir):
              confdir = os.path.join(artifactdir, conf)
              for f in os.listdir(confdir):
                # TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a
                # conf whitelist
                genmap.add(org, confdir).append(f)
                genmap.add((org, name), confdir).append(f)

                genmap.add(target, confdir).append(f)
                genmap.add((target, conf), confdir).append(f)
                genmap.add((org, name, conf), confdir).append(f)
Example #48
0
  def exec_ivy(self,
               target_workdir,
               targets,
               args,
               executor=None,
               confs=None,
               ivy=None,
               workunit_name='ivy',
               jars=None,
               use_soft_excludes=False,
               resolve_hash_name=None):
    ivy_jvm_options = copy.copy(self.get_options().jvm_options)
    # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng.
    ivy_jvm_options.append('-Dsun.io.useCanonCaches=false')

    ivy = ivy or Bootstrapper.default_ivy()
    ivyxml = os.path.join(target_workdir, 'ivy.xml')

    if not jars:
      automatic_excludes = self.get_options().automatic_excludes
      jars, excludes = IvyUtils.calculate_classpath(targets,
                                                    gather_excludes=not use_soft_excludes,
                                                    automatic_excludes=automatic_excludes)
    else:
      excludes = set()

    ivy_args = ['-ivy', ivyxml]

    confs_to_resolve = confs or ['default']
    ivy_args.append('-confs')
    ivy_args.extend(confs_to_resolve)
    ivy_args.extend(args)

    with IvyUtils.ivy_lock:
      IvyUtils.generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve, resolve_hash_name)
      runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor)
      try:
        result = execute_runner(runner, workunit_factory=self.context.new_workunit,
                                workunit_name=workunit_name)
        if result != 0:
          raise TaskError('Ivy returned {result}. cmd={cmd}'.format(result=result, cmd=runner.cmd))
      except runner.executor.Error as e:
        raise TaskError(e)
Example #49
0
  def do_resolve(cls, executor, extra_args, ivyxml, jvm_options, workdir_report_paths_by_conf,
                 confs, ivy_cache_dir, ivy_cache_classpath_filename, resolve_hash_name,
                 workunit_factory, workunit_name):
    """Execute Ivy with the given ivy.xml and copies all relevant files into the workdir.

    This method does an Ivy resolve, which may be either a Pants resolve or a Pants fetch depending
    on whether there is an existing frozen resolution.

    After it is run, the Ivy reports are copied into the workdir at the paths specified by
    workdir_report_paths_by_conf along with a file containing a list of all the requested artifacts
    and their transitive dependencies.

    :param executor: A JVM executor to use to invoke ivy.
    :param extra_args: Extra arguments to pass to ivy.
    :param ivyxml: The input ivy.xml containing the dependencies to resolve.
    :param jvm_options: A list of jvm option strings to use for the ivy invoke, or None.
    :param workdir_report_paths_by_conf: A dict mapping confs to report paths in the workdir.
    :param confs: The confs used in the resolve.
    :param resolve_hash_name: The hash to use as the module name for finding the ivy report file.
    :param workunit_factory: A workunit factory for the ivy invoke, or None.
    :param workunit_name: A workunit name for the ivy invoke, or None.
    """
    ivy = Bootstrapper.default_ivy(bootstrap_workunit_factory=workunit_factory)

    with safe_concurrent_creation(ivy_cache_classpath_filename) as raw_target_classpath_file_tmp:
      extra_args = extra_args or []
      args = ['-cachepath', raw_target_classpath_file_tmp] + extra_args

      with cls._ivy_lock:
        cls._exec_ivy(ivy, confs, ivyxml, args,
                      jvm_options=jvm_options,
                      executor=executor,
                      workunit_name=workunit_name,
                      workunit_factory=workunit_factory)

      if not os.path.exists(raw_target_classpath_file_tmp):
        raise cls.IvyError('Ivy failed to create classpath file at {}'
                           .format(raw_target_classpath_file_tmp))

      cls._copy_ivy_reports(workdir_report_paths_by_conf, confs, ivy_cache_dir, resolve_hash_name)

    logger.debug('Moved ivy classfile file to {dest}'
                 .format(dest=ivy_cache_classpath_filename))
Example #50
0
    def mapjars(self, genmap, target, executor, workunit_factory=None):
        """
    Parameters:
      genmap: the jar_dependencies ProductMapping entry for the required products.
      target: the target whose jar dependencies are being retrieved.
    """
        mapdir = os.path.join(self.mapto_dir(), target.id)
        safe_mkdir(mapdir, clean=True)
        ivyargs = [
            '-retrieve',
            '%s/[organisation]/[artifact]/[conf]/'
            '[organisation]-[artifact]-[revision](-[classifier]).[ext]' %
            mapdir,
            '-symlink',
        ]
        self.exec_ivy(mapdir, [target],
                      ivyargs,
                      confs=target.payload.configurations,
                      ivy=Bootstrapper.default_ivy(executor),
                      workunit_factory=workunit_factory,
                      workunit_name='map-jars')

        for org in os.listdir(mapdir):
            orgdir = os.path.join(mapdir, org)
            if os.path.isdir(orgdir):
                for name in os.listdir(orgdir):
                    artifactdir = os.path.join(orgdir, name)
                    if os.path.isdir(artifactdir):
                        for conf in os.listdir(artifactdir):
                            confdir = os.path.join(artifactdir, conf)
                            for f in os.listdir(confdir):
                                if self.is_mappable_artifact(org, name, f):
                                    # TODO(John Sirois): kill the org and (org, name) exclude mappings in favor of a
                                    # conf whitelist
                                    genmap.add(org, confdir).append(f)
                                    genmap.add((org, name), confdir).append(f)

                                    genmap.add(target, confdir).append(f)
                                    genmap.add((target, conf),
                                               confdir).append(f)
                                    genmap.add((org, name, conf),
                                               confdir).append(f)
Example #51
0
  def publish(self, ivyxml_path, jar, entry, repo, published):
    """Run ivy to publish a jar.  ivyxml_path is the path to the ivy file; published
    is a list of jars published so far (including this one). entry is a pushdb entry."""
    jvm_options = self._ivy_jvm_options(repo)
    resolver = repo['resolver']
    path = repo.get('path')

    try:
      ivy = Bootstrapper.default_ivy()
    except Bootstrapper.Error as e:
      raise TaskError('Failed to push {0}! {1}'.format(pushdb_coordinate(jar, entry), e))

    ivysettings = self.generate_ivysettings(ivy, published, publish_local=path)
    args = [
      '-settings', ivysettings,
      '-ivy', ivyxml_path,
      '-deliverto', '%s/[organisation]/[module]/ivy-[revision].xml' % self.workdir,
      '-publish', resolver,
      '-publishpattern', '%s/[organisation]/[module]/'
                         '[artifact]-[revision](-[classifier]).[ext]' % self.workdir,
      '-revision', entry.version().version(),
      '-m2compatible',
    ]

    # TODO(John Sirois): global logging options should be hidden behind some sort of log manager
    # that we can:
    # a.) obtain a handle to (dependency injection or manual plumbing)
    # b.) query for log detail, ie: `if log_manager.is_verbose:`
    if self.get_options().level == 'debug':
      args.append('-verbose')

    if self.local_snapshot:
      args.append('-overwrite')

    try:
      ivy.execute(jvm_options=jvm_options, args=args,
                  workunit_factory=self.context.new_workunit, workunit_name='jar-publish')
    except Ivy.Error as e:
      raise TaskError('Failed to push {0}! {1}'.format(pushdb_coordinate(jar, entry), e))
Example #52
0
 def test_default_ivy(self):
     ivy = Bootstrapper.default_ivy()
     self.assertIsNotNone(ivy.ivy_cache_dir)
     self.assertIsNone(ivy.ivy_settings)
Example #53
0
 def test_reset(self):
     bootstrapper1 = Bootstrapper.instance()
     Bootstrapper.reset_instance()
     bootstrapper2 = Bootstrapper.instance()
     self.assertNotEqual(bootstrapper1, bootstrapper2)
Example #54
0
  def ivy_resolve(self,
                  targets,
                  executor=None,
                  silent=False,
                  workunit_name=None,
                  confs=None,
                  custom_args=None):
    """Executes an ivy resolve for the relevant subset of the given targets.

    :returns: the resulting classpath, and the unique part of the name used for the resolution
    report (a hash). Also populates the 'ivy_resolve_symlink_map' product for jars resulting
    from the resolve."""

    if not targets:
      return ([], None)

    ivy = Bootstrapper.default_ivy(bootstrap_workunit_factory=self.context.new_workunit)

    ivy_workdir = os.path.join(self.context.options.for_global_scope().pants_workdir, 'ivy')

    fingerprint_strategy = IvyResolveFingerprintStrategy(confs)

    with self.invalidated(targets,
                          invalidate_dependents=False,
                          silent=silent,
                          fingerprint_strategy=fingerprint_strategy) as invalidation_check:
      if not invalidation_check.all_vts:
        return ([], None)
      global_vts = VersionedTargetSet.from_versioned_targets(invalidation_check.all_vts)

      # If a report file is not present, we need to exec ivy, even if all the individual
      # targets up to date... See https://rbcommons.com/s/twitter/r/2015
      report_missing = False
      report_confs = confs or ['default']
      report_paths = []
      resolve_hash_name = global_vts.cache_key.hash
      for conf in report_confs:
        report_path = IvyUtils.xml_report_path(resolve_hash_name, conf)
        if not os.path.exists(report_path):
          report_missing = True
          break
        else:
          report_paths.append(report_path)
      target_workdir = os.path.join(ivy_workdir, resolve_hash_name)
      target_classpath_file = os.path.join(target_workdir, 'classpath')
      raw_target_classpath_file = target_classpath_file + '.raw'
      raw_target_classpath_file_tmp = raw_target_classpath_file + '.tmp'
      # A common dir for symlinks into the ivy2 cache. This ensures that paths to jars
      # in artifact-cached analysis files are consistent across systems.
      # Note that we have one global, well-known symlink dir, again so that paths are
      # consistent across builds.
      symlink_dir = os.path.join(ivy_workdir, 'jars')

      # Note that it's possible for all targets to be valid but for no classpath file to exist at
      # target_classpath_file, e.g., if we previously built a superset of targets.
      if report_missing or invalidation_check.invalid_vts or not os.path.exists(raw_target_classpath_file):
        args = ['-cachepath', raw_target_classpath_file_tmp] + (custom_args if custom_args else [])

        self.exec_ivy(
            target_workdir=target_workdir,
            targets=global_vts.targets,
            args=args,
            executor=executor,
            ivy=ivy,
            workunit_name=workunit_name,
            confs=confs,
            use_soft_excludes=self.get_options().soft_excludes,
            resolve_hash_name=resolve_hash_name)

        if not os.path.exists(raw_target_classpath_file_tmp):
          raise TaskError('Ivy failed to create classpath file at {}'
                          .format(raw_target_classpath_file_tmp))
        shutil.move(raw_target_classpath_file_tmp, raw_target_classpath_file)
        logger.debug('Moved ivy classfile file to {dest}'.format(dest=raw_target_classpath_file))

        if self.artifact_cache_writes_enabled():
          self.update_artifact_cache([(global_vts, [raw_target_classpath_file])])
      else:
        logger.debug("Using previously resolved reports: {}".format(report_paths))

    # Make our actual classpath be symlinks, so that the paths are uniform across systems.
    # Note that we must do this even if we read the raw_target_classpath_file from the artifact
    # cache. If we cache the target_classpath_file we won't know how to create the symlinks.
    with IvyTaskMixin.symlink_map_lock:
      products = self.context.products
      existing_symlinks_map = products.get_data('ivy_resolve_symlink_map', lambda: dict())
      symlink_map = IvyUtils.symlink_cachepath(
        IvySubsystem.global_instance().get_options().cache_dir,
        raw_target_classpath_file,
        symlink_dir,
        target_classpath_file,
        existing_symlinks_map)
      existing_symlinks_map.update(symlink_map)

    with IvyUtils.cachepath(target_classpath_file) as classpath:
      stripped_classpath = [path.strip() for path in classpath]
      return (stripped_classpath, resolve_hash_name)
Example #55
0
 def test_simple(self):
     ivy_subsystem = IvySubsystem.global_instance()
     bootstrapper = Bootstrapper(ivy_subsystem=ivy_subsystem)
     ivy = bootstrapper.ivy()
     self.assertIsNotNone(ivy.ivy_resolution_cache_dir)
     self.assertIsNone(ivy.ivy_settings)
Example #56
0
    def _execute_junit_runner(self,
                              list_of_filename_content_tuples,
                              create_some_resources=True,
                              target_name=None):
        # Create the temporary base test directory
        test_rel_path = 'tests/java/org/pantsbuild/foo'
        test_abs_path = self.create_dir(test_rel_path)

        # Create the temporary classes directory under work dir
        test_classes_abs_path = self.create_workdir_dir(test_rel_path)

        test_java_file_abs_paths = []
        # Generate the temporary java test source code.
        for filename, content in list_of_filename_content_tuples:
            test_java_file_rel_path = os.path.join(test_rel_path, filename)
            test_java_file_abs_path = self.create_file(test_java_file_rel_path,
                                                       content)
            test_java_file_abs_paths.append(test_java_file_abs_path)

        # Invoke ivy to resolve classpath for junit.
        classpath_file_abs_path = os.path.join(test_abs_path,
                                               'junit.classpath')
        ivy_subsystem = global_subsystem_instance(IvySubsystem)
        distribution = DistributionLocator.cached(jdk=True)
        ivy = Bootstrapper(ivy_subsystem=ivy_subsystem).ivy()
        ivy.execute(args=[
            '-cachepath', classpath_file_abs_path, '-dependency', 'junit',
            'junit-dep', '4.10'
        ],
                    executor=SubprocessExecutor(distribution=distribution))

        with open(classpath_file_abs_path) as fp:
            classpath = fp.read()

        # Now directly invoke javac to compile the test java code into classfiles that we can later
        # inject into a product mapping for JUnitRun to execute against.
        javac = distribution.binary('javac')
        subprocess.check_call(
            [javac, '-d', test_classes_abs_path, '-cp', classpath] +
            test_java_file_abs_paths)

        # If a target_name is specified create a target with it, otherwise create a junit_tests target.
        if target_name:
            target = self.target(target_name)
        else:
            target = self.create_library(test_rel_path, 'junit_tests',
                                         'foo_test', ['FooTest.java'])

        target_roots = []
        if create_some_resources:
            # Create a synthetic resource target.
            target_roots.append(self.make_target('some_resources', Resources))
        target_roots.append(target)

        # Set the context with the two targets, one junit_tests target and
        # one synthetic resources target.
        # The synthetic resources target is to make sure we won't regress
        # in the future with bug like https://github.com/pantsbuild/pants/issues/508. Note
        # in that bug, the resources target must be the first one in the list.
        context = self.context(target_roots=target_roots)

        # Before we run the task, we need to inject the "runtime_classpath" with
        # the compiled test java classes that JUnitRun will know which test
        # classes to execute. In a normal run, this "runtime_classpath" will be
        # populated by java compilation step.
        self.populate_runtime_classpath(context=context,
                                        classpath=[test_classes_abs_path])

        # Finally execute the task.
        self.execute(context)