Ejemplo n.º 1
0
def select_libc_objects(platform: Platform, native_toolchain: NativeToolchain) -> LibcObjects:
  # We use lambdas here to avoid searching for libc on osx, where it will fail.
  paths = platform.match({
    Platform.darwin: lambda: [],
    Platform.linux: lambda: native_toolchain._libc_dev.get_libc_objects(),
  })()
  yield LibcObjects(paths)
Ejemplo n.º 2
0
    def test_gcc_version(self):
        scheduler = self._sched()

        platform = Platform.create()

        gcc_subsystem = global_subsystem_instance(GCC)
        gcc_version = gcc_subsystem.version()

        gcc_c_compiler = self.execute_expecting_one_result(
            scheduler, GCCCCompiler, self.toolchain).value
        gcc = gcc_c_compiler.c_compiler
        gcc_version_out = self._invoke_capturing_output(
            [gcc.exe_filename, '--version'],
            env=gcc.get_invocation_environment_dict(platform))

        gcc_version_regex = re.compile('^gcc.*{}$'.format(
            re.escape(gcc_version)),
                                       flags=re.MULTILINE)
        self.assertIsNotNone(gcc_version_regex.search(gcc_version_out))

        gcc_cpp_compiler = self.execute_expecting_one_result(
            scheduler, GCCCppCompiler, self.toolchain).value
        gpp = gcc_cpp_compiler.cpp_compiler
        gpp_version_out = self._invoke_capturing_output(
            [gpp.exe_filename, '--version'],
            env=gpp.get_invocation_environment_dict(platform))

        gpp_version_regex = re.compile(r'^g\+\+.*{}$'.format(
            re.escape(gcc_version)),
                                       flags=re.MULTILINE)
        self.assertIsNotNone(gpp_version_regex.search(gpp_version_out))
Ejemplo n.º 3
0
    def test_clang_version(self):
        scheduler = self._sched()

        platform = Platform.create()

        llvm_subsystem = global_subsystem_instance(LLVM)
        llvm_version = llvm_subsystem.version()
        llvm_version_regex = re.compile('^clang version {}'.format(
            re.escape(llvm_version)),
                                        flags=re.MULTILINE)

        llvm_c_compiler = self.execute_expecting_one_result(
            scheduler, LLVMCCompiler, self.toolchain).value
        clang = llvm_c_compiler.c_compiler
        llvm_version_out = self._invoke_capturing_output(
            [clang.exe_filename, '--version'],
            env=clang.get_invocation_environment_dict(platform))

        self.assertIsNotNone(llvm_version_regex.search(llvm_version_out))

        llvm_cpp_compiler = self.execute_expecting_one_result(
            scheduler, LLVMCppCompiler, self.toolchain).value
        clangpp = llvm_cpp_compiler.cpp_compiler
        gpp_version_out = self._invoke_capturing_output(
            [clangpp.exe_filename, '--version'],
            env=clangpp.get_invocation_environment_dict(platform))

        self.assertIsNotNone(llvm_version_regex.search(gpp_version_out))
Ejemplo n.º 4
0
 def _conan_os_name(self):
     return Platform.create().resolve_platform_specific({
         'darwin':
         lambda: 'Macos',
         'linux':
         lambda: 'Linux',
     })
 def _get_current_platform_string(self):
     return Platform.create().resolve_platform_specific({
         'darwin':
         lambda: 'macosx-10.12-x86_64',
         'linux':
         lambda: 'linux-x86_64',
     })
 def test_build_conan_cmdline_args(self):
   pkg_spec = 'test/1.0.0@conan/stable'
   cr = ConanRequirement(pkg_spec=pkg_spec)
   platform = Platform.create()
   conan_os_name = platform.resolve_platform_specific(self.CONAN_OS_NAME)
   expected = ['install', 'test/1.0.0@conan/stable', '-s', 'os={}'.format(conan_os_name)]
   self.assertEqual(cr.fetch_cmdline_args, expected)
Ejemplo n.º 7
0
def select_libc_objects(platform: Platform, native_toolchain: NativeToolchain) -> LibcObjects:
  # We use lambdas here to avoid searching for libc on osx, where it will fail.
  paths = platform.resolve_for_enum_variant({
    'darwin': lambda: [],
    'linux': lambda: native_toolchain._libc_dev.get_libc_objects(),
  })()
  yield LibcObjects(paths)
Ejemplo n.º 8
0
    def _assert_ctypes_third_party_integration(self, toolchain_variant):
        pants_binary = self.run_pants(
            ['binary', self._binary_target_with_third_party],
            config={
                'native-build-settings': {
                    'toolchain_variant': toolchain_variant,
                },
            })
        self.assert_success(pants_binary)

        # TODO(#6848): this fails when run with gcc on osx as it requires gcc's libstdc++.so.6.dylib to
        # be available on the runtime library path.
        attempt_pants_run = Platform.create().resolve_platform_specific({
            'darwin':
            lambda: toolchain_variant != 'gnu',
            'linux':
            lambda: True,
        })
        if attempt_pants_run:
            pants_run = self.run_pants(
                ['-q', 'run', self._binary_target_with_third_party],
                config={
                    'native-build-settings': {
                        'toolchain_variant': toolchain_variant,
                    },
                })
            self.assert_success(pants_run)
            self.assertIn('Test worked!\n', pants_run.stdout_data)
 def test_build_conan_cmdline_args(self):
     pkg_spec = 'test/1.0.0@conan/stable'
     cr = ConanRequirement(pkg_spec=pkg_spec)
     platform = Platform.create()
     conan_os_name = platform.resolve_platform_specific(self.CONAN_OS_NAME)
     expected = [
         'install', 'test/1.0.0@conan/stable', '-s',
         'os={}'.format(conan_os_name)
     ]
     self.assertEqual(cr.fetch_cmdline_args, expected)
Ejemplo n.º 10
0
  def setUp(self):
    init_subsystems([LibcDev], options={
      'libc': {
        'enable_libc_search': False,
        'libc_dir': '/does/not/exist',
      },
    })

    self.libc = global_subsystem_instance(LibcDev)
    self.platform = Platform.create()
Ejemplo n.º 11
0
  def setUp(self):
    init_subsystems([LibcDev], options={
      'libc': {
        'enable_libc_search': True,
        'host_compiler': 'this_executable_does_not_exist',
      },
    })

    self.libc = global_subsystem_instance(LibcDev)
    self.platform = Platform.create()
Ejemplo n.º 12
0
    def _assert_ctypes_interop_with_mock_buildroot(self, toolchain_variant):
        # TODO: consider making this mock_buildroot/run_pants_with_workdir into a
        # PantsRunIntegrationTest method!
        with self.mock_buildroot(
                dirs_to_copy=[self._binary_interop_target_dir
                              ]) as buildroot, buildroot.pushd():

            # Replace strict_deps=False with nothing so we can override it (because target values for this
            # option take precedence over subsystem options).
            orig_wrapped_math_build = read_file(self._wrapped_math_build_file,
                                                binary_mode=False)
            without_strict_deps_wrapped_math_build = re.sub(
                'strict_deps=False,', '', orig_wrapped_math_build)
            safe_file_dump(self._wrapped_math_build_file,
                           without_strict_deps_wrapped_math_build,
                           mode='w')

            # This should fail because it does not turn on strict_deps for a target which requires it.
            pants_binary_strict_deps_failure = self.run_pants_with_workdir(
                command=['binary', self._binary_target_with_interop],
                # Explicitly set to True (although this is the default).
                config={
                    # TODO(#6848): don't make it possible to forget to add the toolchain_variant option!
                    'native-build-settings': {
                        'toolchain_variant': toolchain_variant,
                        'strict_deps': True,
                    },
                },
                workdir=os.path.join(buildroot.new_buildroot, '.pants.d'),
                build_root=buildroot.new_buildroot)
            self.assert_failure(pants_binary_strict_deps_failure)
            self.assertIn(
                self._include_not_found_message_for_variant[toolchain_variant],
                pants_binary_strict_deps_failure.stdout_data)

        # TODO(#6848): we need to provide the libstdc++.so.6.dylib which comes with gcc on osx in the
        # DYLD_LIBRARY_PATH during the 'run' goal somehow.
        attempt_pants_run = Platform.create().resolve_platform_specific({
            'darwin':
            lambda: toolchain_variant != 'gnu',
            'linux':
            lambda: True,
        })
        if attempt_pants_run:
            pants_run_interop = self.run_pants(
                ['-q', 'run', self._binary_target_with_interop],
                config={
                    'native-build-settings': {
                        'toolchain_variant': toolchain_variant,
                        'strict_deps': False,
                    },
                })
            self.assert_success(pants_run_interop)
            self.assertEqual('x=3, f(x)=299\n', pants_run_interop.stdout_data)
Ejemplo n.º 13
0
  def _assert_ctypes_binary_creation(self, toolchain_variant):
    with temporary_dir() as tmp_dir:
      pants_run = self.run_pants(command=['binary', self._binary_target], config={
        GLOBAL_SCOPE_CONFIG_SECTION: {
          'pants_distdir': tmp_dir,
        },
        'native-build-step': {
          'toolchain_variant': toolchain_variant,
        },
      })

      self.assert_success(pants_run)

      # Check that we have selected the appropriate compilers for our selected toolchain variant,
      # for both C and C++ compilation.
      # TODO(#6866): don't parse info logs for testing!
      for compiler_name in self._compiler_names_for_variant[toolchain_variant]:
        self.assertIn("selected compiler exe name: '{}'".format(compiler_name),
                      pants_run.stdout_data)

      for linker_name in self._linker_names_for_variant[toolchain_variant]:
        self.assertIn("selected linker exe name: '{}'".format(linker_name),
                      pants_run.stdout_data)

      # Check for the pex and for the wheel produced for our python_dist().
      pex = os.path.join(tmp_dir, 'bin.pex')
      self.assertTrue(is_executable(pex))

      # The + is because we append the target's fingerprint to the version. We test this version
      # string in test_build_local_python_distributions.py.
      wheel_glob = os.path.join(tmp_dir, 'ctypes_test-0.0.1+*.whl')
      wheel_dist_with_path = assert_single_element(glob.glob(wheel_glob))
      wheel_dist = re.sub('^{}{}'.format(re.escape(tmp_dir), os.path.sep), '', wheel_dist_with_path)

      dist_name, dist_version, wheel_platform = name_and_platform(wheel_dist)
      self.assertEqual(dist_name, 'ctypes_test')
      contains_current_platform = Platform.create().resolve_platform_specific({
        'darwin': lambda: wheel_platform.startswith('macosx'),
        'linux': lambda: wheel_platform.startswith('linux'),
      })
      self.assertTrue(contains_current_platform)

      # Verify that the wheel contains our shared libraries.
      wheel_files = ZipFile(wheel_dist_with_path).namelist()

      dist_versioned_name = '{}-{}.data'.format(dist_name, dist_version)
      for shared_lib_filename in ['libasdf-c_ctypes.so', 'libasdf-cpp_ctypes.so']:
        full_path_in_wheel = os.path.join(dist_versioned_name, 'data', shared_lib_filename)
        self.assertIn(full_path_in_wheel, wheel_files)

      # Execute the binary and ensure its output is correct.
      binary_run_output = invoke_pex_for_output(pex)
      self.assertEqual(b'x=3, f(x)=17\n', binary_run_output)
Ejemplo n.º 14
0
    def setUp(self):
        init_subsystems(
            [LibcDev],
            options={
                'libc': {
                    'enable_libc_search': False,
                    'libc_dir': '/does/not/exist',
                },
            })

        self.libc = global_subsystem_instance(LibcDev)
        self.platform = Platform.create()
Ejemplo n.º 15
0
    def setUp(self):
        init_subsystems(
            [LibcDev],
            options={
                'libc': {
                    'enable_libc_search': True,
                    'host_compiler': 'this_executable_does_not_exist',
                },
            })

        self.libc = global_subsystem_instance(LibcDev)
        self.platform = Platform.create()
Ejemplo n.º 16
0
    def setUp(self):
        super(TestNativeToolchain, self).setUp()

        init_subsystems([LibcDev, NativeToolchain],
                        options={
                            'libc': {
                                'enable_libc_search': True,
                            },
                        })

        self.platform = Platform.create()
        self.toolchain = global_subsystem_instance(NativeToolchain)
        self.rules = native_backend_rules()
Ejemplo n.º 17
0
    def _compile(self, compile_request):
        """Perform the process of compilation, writing object files to the request's 'output_dir'.

    NB: This method must arrange the output files so that `collect_cached_objects()` can collect all
    of the results (or vice versa)!
    """
        sources = compile_request.sources

        if len(sources) == 0:
            # TODO: do we need this log message? Should we still have it for intentionally header-only
            # libraries (that might be a confusing message to see)?
            self.context.log.debug(
                "no sources in request {}, skipping".format(compile_request))
            return

        compiler = compile_request.compiler
        output_dir = compile_request.output_dir

        argv = self._make_compile_argv(compile_request)

        platform = Platform.create()

        with self.context.new_workunit(name=self.workunit_label,
                                       labels=[WorkUnitLabel.COMPILER
                                               ]) as workunit:
            try:
                process = subprocess.Popen(
                    argv,
                    cwd=output_dir,
                    stdout=workunit.output('stdout'),
                    stderr=workunit.output('stderr'),
                    env=compiler.get_invocation_environment_dict(platform))
            except OSError as e:
                workunit.set_outcome(WorkUnit.FAILURE)
                raise self.NativeCompileError(
                    "Error invoking '{exe}' with command {cmd} for request {req}: {err}"
                    .format(exe=compiler.exe_filename,
                            cmd=argv,
                            req=compile_request,
                            err=e))

            rc = process.wait()
            if rc != 0:
                workunit.set_outcome(WorkUnit.FAILURE)
                raise self.NativeCompileError(
                    "Error in '{section_name}' with command {cmd} for request {req}. Exit code was: {rc}."
                    .format(section_name=self.workunit_name,
                            cmd=argv,
                            req=compile_request,
                            rc=rc))
Ejemplo n.º 18
0
    def test_ctypes_binary(self):
        with temporary_dir() as tmp_dir:
            pants_run = self.run_pants(command=['binary', self._binary_target],
                                       config={
                                           GLOBAL_SCOPE_CONFIG_SECTION: {
                                               'pants_distdir': tmp_dir,
                                           }
                                       })

            self.assert_success(pants_run)

            # Check for the pex and for the wheel produced for our python_dist().
            pex = os.path.join(tmp_dir, 'bin.pex')
            self.assertTrue(is_executable(pex))

            # The + is because we append the target's fingerprint to the version. We test this version
            # string in test_build_local_python_distributions.py.
            wheel_glob = os.path.join(tmp_dir, 'ctypes_test-0.0.1+*.whl')
            wheel_dist_with_path = assert_single_element(glob.glob(wheel_glob))
            wheel_dist = re.sub(
                '^{}{}'.format(re.escape(tmp_dir), os.path.sep), '',
                wheel_dist_with_path)

            dist_name, dist_version, wheel_platform = name_and_platform(
                wheel_dist)
            self.assertEqual(dist_name, 'ctypes_test')
            contains_current_platform = Platform.create(
            ).resolve_platform_specific({
                'darwin':
                lambda: wheel_platform.startswith('macosx'),
                'linux':
                lambda: wheel_platform.startswith('linux'),
            })
            self.assertTrue(contains_current_platform)

            # Verify that the wheel contains our shared libraries.
            wheel_files = ZipFile(wheel_dist_with_path).namelist()

            dist_versioned_name = '{}-{}.data'.format(dist_name, dist_version)
            for shared_lib_filename in ['libasdf-c.so', 'libasdf-cpp.so']:
                full_path_in_wheel = os.path.join(dist_versioned_name, 'data',
                                                  shared_lib_filename)
                self.assertIn(full_path_in_wheel, wheel_files)

            # Execute the binary and ensure its output is correct.
            binary_run_output = invoke_pex_for_output(pex)
            self.assertEqual('x=3, f(x)=17\n', binary_run_output)
Ejemplo n.º 19
0
  def setUp(self):
    super(TestNativeToolchain, self).setUp()

    init_subsystems([LibcDev, NativeToolchain], options={
      'libc': {
        'enable_libc_search': True,
      },
    })

    self.platform = Platform.create()
    self.toolchain = global_subsystem_instance(NativeToolchain)
    self.rules = native_backend_rules()

    gcc_subsystem = global_subsystem_instance(GCC)
    self.gcc_version = gcc_subsystem.version()
    llvm_subsystem = global_subsystem_instance(LLVM)
    self.llvm_version = llvm_subsystem.version()
Ejemplo n.º 20
0
    def _execute_link_request(self, link_request):
        object_files = link_request.object_files

        if len(object_files) == 0:
            raise self.LinkSharedLibrariesError(
                "No object files were provided in request {}!".format(
                    link_request))

        platform = Platform.create()
        linker = link_request.linker
        native_artifact = link_request.native_artifact
        output_dir = link_request.output_dir
        resulting_shared_lib_path = os.path.join(
            output_dir, native_artifact.as_shared_lib(platform))
        # We are executing in the results_dir, so get absolute paths for everything.
        cmd = ([linker.exe_filename] +
               self._get_shared_lib_cmdline_args(platform) +
               ['-o', os.path.abspath(resulting_shared_lib_path)] +
               [os.path.abspath(obj) for obj in object_files])

        with self.context.new_workunit(name='link-shared-libraries',
                                       labels=[WorkUnitLabel.LINKER
                                               ]) as workunit:
            try:
                process = subprocess.Popen(
                    cmd,
                    cwd=output_dir,
                    stdout=workunit.output('stdout'),
                    stderr=workunit.output('stderr'),
                    env=linker.get_invocation_environment_dict(platform))
            except OSError as e:
                workunit.set_outcome(WorkUnit.FAILURE)
                raise self.LinkSharedLibrariesError(
                    "Error invoking the native linker with command {} for request {}: {}"
                    .format(cmd, link_request, e), e)

            rc = process.wait()
            if rc != 0:
                workunit.set_outcome(WorkUnit.FAILURE)
                raise self.LinkSharedLibrariesError(
                    "Error linking native objects with command {} for request {}. Exit code was: {}."
                    .format(cmd, link_request, rc))

        return SharedLibrary(name=native_artifact.lib_name,
                             path=resulting_shared_lib_path)
Ejemplo n.º 21
0
 def test_pants_tests_local_dists_for_current_platform_only(self):
     platform_string = Platform.create().resolve_platform_specific({
         'darwin':
         lambda: 'macosx-10.12-x86_64',
         'linux':
         lambda: 'linux-x86_64',
     })
     # Use a platform-specific string for testing because the test goal
     # requires the coverage package and the pex resolver raises an Untranslatable error when
     # attempting to translate the coverage sdist for incompatible platforms.
     pants_ini_config = {'python-setup': {'platforms': [platform_string]}}
     # Clean all to rebuild requirements pex.
     with temporary_dir() as tmp_dir:
         command = [
             '--pants-distdir={}'.format(tmp_dir), 'clean-all', 'test',
             '{}:fasthello'.format(self.fasthello_tests)
         ]
         pants_run = self.run_pants(command=command,
                                    config=pants_ini_config)
         self.assert_success(pants_run)
Ejemplo n.º 22
0
    def _make_compile_argv(self, compile_request):
        """Return a list of arguments to use to compile sources. Subclasses can override and append."""
        compiler = compile_request.compiler
        err_flags = ['-Werror'] if compile_request.fatal_warnings else []

        platform = Platform.create()

        platform_specific_flags = platform.resolve_platform_specific({
            'linux':
            lambda: [],
            'darwin':
            lambda: ['-mmacosx-version-min=10.11'],
        })

        # We are going to execute in the target output, so get absolute paths for everything.
        # TODO: If we need to produce static libs, don't add -fPIC! (could use Variants -- see #5788).
        argv = ([compiler.exe_filename] + platform_specific_flags +
                self.extra_compile_args() + err_flags + ['-c', '-fPIC'] + [
                    '-I{}'.format(os.path.abspath(inc_dir))
                    for inc_dir in compile_request.include_dirs
                ] + [os.path.abspath(src) for src in compile_request.sources])

        return argv
Ejemplo n.º 23
0
  def test_binary(self):
    with temporary_dir() as tmp_dir:
      pants_run = self.run_pants(command=['binary', self._binary_target], config={
        GLOBAL_SCOPE_CONFIG_SECTION: {
          'pants_distdir': tmp_dir,
        }
      })

      self.assert_success(pants_run)

      # Check for the pex and for the wheel produced for our python_dist().
      pex = os.path.join(tmp_dir, 'bin.pex')
      self.assertTrue(is_executable(pex))

      wheel_glob = os.path.join(tmp_dir, 'ctypes_test-0.0.1-*.whl')
      globbed_wheel = glob.glob(wheel_glob)
      self.assertEqual(len(globbed_wheel), 1)
      wheel_dist = globbed_wheel[0]

      _, _, wheel_platform = name_and_platform(wheel_dist)
      contains_current_platform = Platform.create().resolve_platform_specific({
        'darwin': lambda: wheel_platform.startswith('macosx'),
        'linux': lambda: wheel_platform.startswith('linux'),
      })
      self.assertTrue(contains_current_platform)

      # Verify that the wheel contains our shared libraries.
      wheel_files = ZipFile(wheel_dist).namelist()

      for shared_lib_filename in ['libasdf-c.so', 'libasdf-cpp.so']:
        full_path_in_wheel = os.path.join('ctypes_test-0.0.1.data', 'data', shared_lib_filename)
        self.assertIn(full_path_in_wheel, wheel_files)

      # Execute the binary and ensure its output is correct.
      binary_run_output = invoke_pex_for_output(pex)
      self.assertEqual('x=3, f(x)=17\n', binary_run_output)
Ejemplo n.º 24
0
 def wrapper(self, *args, **kwargs):
   if Platform.current == Platform(normalized_os_name):
     test_fn(self, *args, **kwargs)
 def _get_current_platform_string(self):
   return Platform.create().resolve_platform_specific({
     'darwin': lambda: 'macosx-10.12-x86_64',
     'linux': lambda: 'linux-x86_64',
   })
Ejemplo n.º 26
0
 def _platform(cls):
   return Platform.create()
Ejemplo n.º 27
0
 def platform(self):
     # TODO: convert this to a v2 engine dependency injection.
     return Platform.create()
Ejemplo n.º 28
0
    def wrapper(self, *args, **kwargs):
      # FIXME: This should be drawn from the v2 engine somehow.
      platform = Platform.create()

      if platform.normalized_os_name == normalized_os_name:
        test_fn(self, *args, **kwargs)
Ejemplo n.º 29
0
 def fetch_cmdline_args(self):
     platform = Platform.create()
     conan_os_name = platform.resolve_platform_specific(self.CONAN_OS_NAME)
     args = ['install', self.pkg_spec, '-s', 'os={}'.format(conan_os_name)]
     return args
Ejemplo n.º 30
0
        def wrapper(self, *args, **kwargs):
            # TODO: This should be drawn from the v2 engine somehow.
            platform = Platform.create()

            if platform.normalized_os_name == normalized_os_name:
                test_fn(self, *args, **kwargs)
Ejemplo n.º 31
0
    def setUp(self):
        super(TestNativeToolchain, self).setUp()

        self.platform = Platform.create()
        self.toolchain = global_subsystem_instance(NativeToolchain)
        self.rules = native_backend_rules()
Ejemplo n.º 32
0
 def _current_platform_abbreviation(cls):
   return NativeBackendPlatform.create().resolve_platform_specific({
     'darwin': lambda: 'macosx',
     'linux': lambda: 'linux',
   })