on_mac, ) from enum import Enum from rust_file import ( CrateType, RustFile, RustFileBuilder, RustFunction, RustMatch, RustMod, RustVisibility, ) from typing import Generator, List, Optional, Set, Iterable # Tools we will need clang = get_cmd_or_die("clang") rustc = get_cmd_or_die("rustc") diff = get_cmd_or_die("diff") ar = get_cmd_or_die("ar") cargo = get_cmd_or_die("cargo") # Intermediate files intermediate_files = [ 'cc_db', 'c_obj', 'c_lib', 'rust_src', ] class TestOutcome(Enum):
import logging import os from enum import Enum from common import get_cmd_or_die, NonZeroReturn from plumbum.machines.local import LocalCommand from typing import Iterable, List, Optional, Tuple rustc = get_cmd_or_die("rustc") # TODO: Support for custom visibility paths, if needed class RustVisibility(Enum): Private = "" Public = "pub " Crate = "pub(crate) " class CrateType(Enum): Binary = "bin" Library = "lib" class RustFile: def __init__(self, path: str) -> None: self.path = path def compile(self, crate_type: CrateType, save_output: bool = False, extra_args: List[str] = []) -> Optional[LocalCommand]:
def transpile_files(cc_db: TextIO, filter: str = None, extra_impo_args: List[str] = [], import_only: bool = False, verbose: bool = False, emit_build_files: bool = True, main_module_for_build_files: str = None, cross_checks: bool = False, use_fakechecks: bool = False, cross_check_config: List[str] = [], reloop_cfgs: bool = True) -> bool: """ run the ast-exporter and ast-importer on all C files in a compile commands database. """ rustfmt = os.path.join(get_rust_toolchain_binpath(), "rustfmt") rustfmt = get_cmd_or_die(rustfmt) ast_expo = get_cmd_or_die(c.AST_EXPO) ast_impo = get_cmd_or_die(c.AST_IMPO) cc_db_name = cc_db.name cc_db = json.load(cc_db) check_main_module(main_module_for_build_files, cc_db) if filter: # skip commands not matching file filter cc_db = [cmd for cmd in cc_db if filter in c['file']] if not on_mac(): ensure_code_compiled_with_clang(cc_db) include_dirs = get_system_include_dirs() impo_args = ['--translate-entry'] if emit_build_files: impo_args.append('--emit-module') if cross_checks: impo_args.append('--cross-checks') for ccc in cross_check_config: impo_args.append('--cross-check-config') impo_args.append(ccc) if reloop_cfgs: impo_args.append('--reloop-cfgs') def transpile_single(cmd) -> Tuple[str, int, str, str, str]: if import_only: cbor_file = os.path.join(cmd['directory'], cmd['file'] + ".cbor") else: cbor_file = export_ast_from(ast_expo, cc_db_name, include_dirs, **cmd) assert os.path.isfile(cbor_file), "missing: " + cbor_file ld_lib_path = get_rust_toolchain_libpath() # don't overwrite existing ld lib path if any... if 'LD_LIBRARY_PATH' in pb.local.env: ld_lib_path += ':' + pb.local.env['LD_LIBRARY_PATH'] # import ast with pb.local.env(RUST_BACKTRACE='1', LD_LIBRARY_PATH=ld_lib_path): file_basename = os.path.basename(cmd['file']) cbor_basename = os.path.basename(cbor_file) logging.info(" importing ast from %s", cbor_basename) translation_cmd = "RUST_BACKTRACE=1 \\\n" translation_cmd += "LD_LIBRARY_PATH=" + ld_lib_path + " \\\n" translation_cmd += str(ast_impo[cbor_file, impo_args, extra_impo_args]) logging.debug("translation command:\n %s", translation_cmd) try: retcode, _stdout, _stderr = ast_impo[cbor_file, impo_args, extra_impo_args].run() e = "Expected file suffix `.c.cbor`; actual: " + cbor_basename assert cbor_file.endswith(".c.cbor"), e rust_file = cbor_file[:-7] + ".rs" with open(rust_file, "w") as rust_fh: rust_fh.writelines(_stdout) logging.debug("wrote output rust to %s", rust_file) rustfmt(rust_file) return (file_basename, retcode, _stdout, _stderr, os.path.abspath(rust_file)) except pb.ProcessExecutionError as pee: return (file_basename, pee.retcode, pee.stdout, pee.stderr, None) commands = sorted(cc_db, key=lambda cmd: os.path.basename(cmd['file'])) results = (transpile_single(cmd) for cmd in commands) if emit_build_files: modules = [(rust_src, retcode == 0) for (_, retcode, _, _, rust_src) in results if rust_src is not None] cc_db_dir = os.path.dirname(cc_db_name) write_build_files(cc_db_dir, modules, main_module_for_build_files, cross_checks, use_fakechecks, cross_check_config) successes, failures = 0, 0 for (fname, retcode, stdout, stderr, _) in results: if not retcode: successes += 1 print(Colors.OKGREEN + " import successful" + Colors.NO_COLOR) logging.debug(" import successful") else: # non-zero retcode failures += 1 if verbose: print(Colors.FAIL + " import failed" + Colors.NO_COLOR) logging.debug(" import failed") logging.warning(stderr) else: print(Colors.FAIL + " import failed (error in log)" + Colors.NO_COLOR) logging.debug(" import failed") logging.debug(stderr) print("translations: " + str(successes + failures)) print("successes...: " + str(successes)) print("failures....: " + str(failures)) return failures == 0
def build_clang_plugin(args: str) -> None: """ run cmake as needed to generate ninja buildfiles. then run ninja. """ cargo = get_cmd_or_die("cargo") config_capi_src_dir = os.path.join(c.CROSS_CHECKS_DIR, "rust-checks", "config-capi") cargo_target_dir = os.path.join(c.CLANG_XCHECK_PLUGIN_BLD, "config-capi-target") config_lib_path = os.path.join(cargo_target_dir, "debug" if args.debug else "release", "libc2rust_xcheck_config_capi.a") with pb.local.cwd(config_capi_src_dir): cargo_args = ["build", "--package", "c2rust-xcheck-config-capi"] if not args.debug: cargo_args.append("--release") with pb.local.env(CARGO_TARGET_DIR=cargo_target_dir): invoke(cargo[cargo_args]) ninja = get_cmd_or_die("ninja") # Possible values are Release, Debug, RelWithDebInfo and MinSizeRel build_type = "Debug" if args.debug else "RelWithDebInfo" ninja_build_file = os.path.join(c.CLANG_XCHECK_PLUGIN_BLD, "build.ninja") with pb.local.cwd(c.CLANG_XCHECK_PLUGIN_BLD): if os.path.isfile(ninja_build_file): prev_build_type = get_ninja_build_type(ninja_build_file) run_cmake = prev_build_type != build_type else: run_cmake = True if run_cmake: cmake = get_cmd_or_die("cmake") max_link_jobs = est_parallel_link_jobs() cargs = [ "-G", "Ninja", c.CLANG_XCHECK_PLUGIN_SRC, "-DXCHECK_CONFIG_LIB={}".format(config_lib_path), "-DCMAKE_BUILD_TYPE=" + build_type, "-DBUILD_SHARED_LIBS=1", "-DLLVM_PARALLEL_LINK_JOBS={}".format(max_link_jobs) ] if args.with_c2rust_clang: llvm_cmake_dir = os.path.join(c.LLVM_BLD, "lib", "cmake", "llvm") if not os.path.exists(llvm_cmake_dir): die("missing LLVM cmake files at: " + llvm_cmake_dir) clang_cmake_dir = os.path.join(c.LLVM_BLD, "lib", "cmake", "clang") if not os.path.exists(clang_cmake_dir): die("missing clang cmake files at: " + clang_cmake_dir) llvm_lit = os.path.join(c.LLVM_BLD, "bin", "llvm-lit") if not os.path.exists(llvm_lit): die("missing llvm-lit binary at: " + llvm_lit) cargs.extend([ "-DLLVM_DIR={}".format(llvm_cmake_dir), "-DClang_DIR={}".format(clang_cmake_dir), "-DLLVM_EXTERNAL_LIT={}".format(llvm_lit) ]) else: # Some distros, e.g., Arch, Ubuntu, ship llvm-lit as /usr/bin/lit cargs.append("-DLLVM_EXTERNAL_LIT={}".format(pb.local['lit'])) invoke(cmake[cargs]) else: logging.debug("found existing ninja.build, not running cmake") invoke(ninja)
def configure_and_build_llvm(args) -> None: """ run cmake as needed to generate ninja buildfiles. then run ninja. """ # Possible values are Release, Debug, RelWithDebInfo and MinSizeRel build_type = "Debug" if args.debug else "RelWithDebInfo" ninja_build_file = os.path.join(c.LLVM_BLD, "build.ninja") with pb.local.cwd(c.LLVM_BLD): if os.path.isfile(ninja_build_file) and not args.xcode: prev_build_type = get_ninja_build_type(ninja_build_file) run_cmake = prev_build_type != build_type else: run_cmake = True if run_cmake: cmake = get_cmd_or_die("cmake") clang = get_cmd_or_die("clang") clangpp = get_cmd_or_die("clang++") max_link_jobs = est_parallel_link_jobs() assertions = "1" if args.assertions else "0" ast_ext_dir = "-DLLVM_EXTERNAL_C2RUST_AST_EXPORTER_SOURCE_DIR={}" ast_ext_dir = ast_ext_dir.format(c.AST_EXPO_SRC_DIR) cargs = ["-G", "Ninja", c.LLVM_SRC, "-Wno-dev", "-DCMAKE_C_COMPILER={}".format(clang), "-DCMAKE_CXX_COMPILER={}".format(clangpp), "-DCMAKE_INSTALL_PREFIX=" + c.LLVM_INSTALL, "-DCMAKE_BUILD_TYPE=" + build_type, "-DLLVM_PARALLEL_LINK_JOBS={}".format(max_link_jobs), "-DLLVM_ENABLE_ASSERTIONS=" + assertions, "-DCMAKE_EXPORT_COMPILE_COMMANDS=1", # required to build LLVM 8 on Debian Jessie "-DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=1", ast_ext_dir] if on_x86(): # speed up builds on x86 hosts cargs.append("-DLLVM_TARGETS_TO_BUILD=X86") invoke(cmake[cargs]) # NOTE: we only generate Xcode project files for IDE support # and don't build with them since the cargo build.rs files # rely on cmake to build native code. if args.xcode: cargs[1] = "Xcode" # output Xcode project files in a separate dir ensure_dir(c.AST_EXPO_PRJ_DIR) with pb.local.cwd(c.AST_EXPO_PRJ_DIR): invoke(cmake[cargs]) else: logging.debug("found existing ninja.build, not running cmake") # if args.xcode: # xcodebuild = get_cmd_or_die("xcodebuild") # xc_conf_args = ['-configuration', build_type] # xc_args = xc_conf_args + ['-target', 'llvm-config'] # invoke(xcodebuild, *xc_args) # xc_args = xc_conf_args + ['-target', 'c2rust-ast-exporter'] # invoke(xcodebuild, *xc_args) # We must install headers here so our clang tool can reference # compiler-internal headers such as stddef.h. This reference is # relative to LLVM_INSTALL/bin, which MUST exist for the relative # reference to be valid. To force this, we also install llvm-config, # since we are building and using it for other purposes. ninja = get_cmd_or_die("ninja") ninja_args = ['c2rust-ast-exporter', 'clangAstExporter', 'llvm-config', 'install-clang-headers', 'FileCheck', 'count', 'not'] if args.with_clang: ninja_args.append('clang') invoke(ninja, *ninja_args) # Make sure install/bin exists so that we can create a relative path # using it in AstExporter.cpp os.makedirs(os.path.join(c.LLVM_INSTALL, 'bin'), exist_ok=True)
def configure_and_build_llvm(args) -> None: """ run cmake as needed to generate ninja buildfiles. then run ninja. """ # Possible values are Release, Debug, RelWithDebInfo and MinSizeRel build_type = "Debug" if args.debug else "RelWithDebInfo" ninja_build_file = os.path.join(c.LLVM_BLD, "build.ninja") with pb.local.cwd(c.LLVM_BLD): if os.path.isfile(ninja_build_file) and not args.xcode: prev_build_type = get_ninja_build_type(ninja_build_file) run_cmake = prev_build_type != build_type else: run_cmake = True if run_cmake: cmake = get_cmd_or_die("cmake") max_link_jobs = est_parallel_link_jobs() assertions = "1" if args.assertions else "0" cargs = ["-G", "Ninja", c.LLVM_SRC, "-Wno-dev", "-DCMAKE_INSTALL_PREFIX=" + c.LLVM_INSTALL, "-DCMAKE_BUILD_TYPE=" + build_type, "-DLLVM_PARALLEL_LINK_JOBS={}".format(max_link_jobs), "-DLLVM_ENABLE_ASSERTIONS=" + assertions, "-DCMAKE_EXPORT_COMPILE_COMMANDS=1", "-DLLVM_TARGETS_TO_BUILD=host"] invoke(cmake[cargs]) # NOTE: we only generate Xcode project files for IDE support # and don't build with them since the cargo build.rs files # rely on cmake to build native code. if args.xcode: cargs[1] = "Xcode" # output Xcode project files in a separate dir ensure_dir(c.AST_EXPO_PRJ_DIR) with pb.local.cwd(c.AST_EXPO_PRJ_DIR): invoke(cmake[cargs]) else: logging.debug("found existing ninja.build, not running cmake") # if args.xcode: # xcodebuild = get_cmd_or_die("xcodebuild") # xc_conf_args = ['-configuration', build_type] # xc_args = xc_conf_args + ['-target', 'llvm-config'] # invoke(xcodebuild, *xc_args) # xc_args = xc_conf_args + ['-target', 'c2rust-ast-exporter'] # invoke(xcodebuild, *xc_args) # We must install headers here so our clang tool can reference # compiler-internal headers such as stddef.h. This reference is # relative to LLVM_INSTALL/bin, which MUST exist for the relative # reference to be valid. To force this, we also install llvm-config, # since we are building and using it for other purposes. nice = get_cmd_or_die("nice") ninja = get_cmd_or_die("ninja") nice_args = [ '-n', '19', str(ninja), 'clangAST', 'clangFrontend', 'clangTooling', 'clangBasic', 'clangASTMatchers', 'clangParse', 'clangSerialization', 'clangSema', 'clangEdit', 'clangAnalysis', 'clangDriver', 'clangFormat', 'clangToolingCore', 'clangRewrite', 'clangLex', 'LLVMMC', 'LLVMMCParser', 'LLVMDemangle', 'LLVMSupport', 'LLVMOption', 'LLVMBinaryFormat', 'LLVMCore', 'LLVMBitReader', 'LLVMProfileData', 'llvm-config', 'install-clang-headers', 'install-compiler-rt-headers', 'FileCheck', 'count', 'not'] (major, _minor, _point) = c.LLVM_VER.split(".") major = int(major) if major >= 7 and major < 10: nice_args += [ 'LLVMDebugInfoMSF', 'LLVMDebugInfoCodeView'] if major > 8: nice_args.append("install-clang-resource-headers") if major == 9: nice_args += [ 'LLVMBitstreamReader', 'LLVMRemarks'] if major >= 10: nice_args.append("LLVMFrontendOpenMP") if args.with_clang: nice_args.append('clang') invoke(nice, *nice_args) # Make sure install/bin exists so that we can create a relative path # using it in AstExporter.cpp os.makedirs(os.path.join(c.LLVM_INSTALL, 'bin'), exist_ok=True)
def transpile_files(cc_db: TextIO, filter: Optional[Callable[[str], bool]] = None, extra_impo_args: List[str] = [], import_only: bool = False, verbose: bool = False, emit_build_files: bool = True, emit_modules: bool = False, main_module_for_build_files: str = None, cross_checks: bool = False, use_fakechecks: bool = False, cross_check_config: List[str] = [], reloop_cfgs: bool = True, reorganize_definitions: bool = False) -> bool: """ run the ast-exporter and ast-importer on all C files in a compile commands database. """ rustfmt = os.path.join(get_rust_toolchain_binpath(), "rustfmt") rustfmt = get_cmd_or_die(rustfmt) ast_expo = get_cmd_or_die(c.AST_EXPO) ast_impo = get_cmd_or_die(c.AST_IMPO) cc_db_name = cc_db.name cc_db = json.load(cc_db) check_main_module(main_module_for_build_files, cc_db) if filter: # skip commands not matching file filter cc_db = [cmd for cmd in cc_db if filter(cmd['file'])] if not on_mac(): ensure_code_compiled_with_clang(cc_db) # MacOS Mojave does not have `/usr/include` even if the command line # tools are installed. The fix is to run the developer package: # `macOS_SDK_headers_for_macOS_10.14.pkg` in # `/Library/Developer/CommandLineTools/Packages`. # Source https://forums.developer.apple.com/thread/104296 if on_mac() and not os.path.isdir('/usr/include'): emsg = ("directory /usr/include not found. " "Please install the following package: " "/Library/Developer/CommandLineTools/Packages/" "macOS_SDK_headers_for_macOS_10.14.pkg " "or the equivalent version on your host.") die(emsg, errno.ENOENT) impo_args = ['--translate-entry'] if emit_build_files or emit_modules: impo_args.append('--emit-module') if cross_checks: impo_args.append('--cross-checks') for ccc in cross_check_config: impo_args.append('--cross-check-config') impo_args.append(ccc) if reloop_cfgs: impo_args.append('--reloop-cfgs') if reorganize_definitions: impo_args.append('--reorganize-definitions') def transpile_single(cmd) -> Tuple[str, int, str, str, str]: if import_only: cbor_file = os.path.join(cmd['directory'], cmd['file'] + ".cbor") else: cbor_file = export_ast_from(ast_expo, cc_db_name, **cmd) assert os.path.isfile(cbor_file), "missing: " + cbor_file ld_lib_path = get_rust_toolchain_libpath() # don't overwrite existing ld lib path if any... if 'LD_LIBRARY_PATH' in pb.local.env: ld_lib_path += ':' + pb.local.env['LD_LIBRARY_PATH'] # import ast with pb.local.env(RUST_BACKTRACE='1', LD_LIBRARY_PATH=ld_lib_path): file_basename = os.path.basename(cmd['file']) cbor_basename = os.path.basename(cbor_file) logging.info(" importing ast from %s", cbor_basename) translation_cmd = "RUST_BACKTRACE=1 \\\n" translation_cmd += "LD_LIBRARY_PATH=" + ld_lib_path + " \\\n" translation_cmd += str(ast_impo[cbor_file, impo_args, extra_impo_args]) logging.debug("translation command:\n %s", translation_cmd) try: ast_impo_cmd = ast_impo[cbor_file, impo_args, extra_impo_args] # NOTE: this will log ast-importer output but not in color retcode, stdout, importer_warnings = ast_impo_cmd.run() if importer_warnings: if verbose: logging.warning(importer_warnings) else: logging.debug(importer_warnings) e = "Expected file suffix `.c.cbor`; actual: " + cbor_basename assert cbor_file.endswith(".c.cbor"), e rust_file = cbor_file[:-7] + ".rs" path, file_name = os.path.split(rust_file) file_name = file_name.replace('-', '_') rust_file = os.path.join(path, file_name) rustfmt(rust_file) return (file_basename, retcode, stdout, importer_warnings, os.path.abspath(rust_file)) except pb.ProcessExecutionError as pee: return (file_basename, pee.retcode, pee.stdout, pee.stderr, None) commands = sorted(cc_db, key=lambda cmd: os.path.basename(cmd['file'])) results = [transpile_single(cmd) for cmd in commands] if emit_build_files: modules = [(rust_src, retcode == 0) for (_, retcode, _, _, rust_src) in results if rust_src is not None] cc_db_dir = os.path.dirname(cc_db_name) write_build_files(cc_db_dir, modules, main_module_for_build_files, cross_checks, reorganize_definitions, use_fakechecks, cross_check_config) successes, failures = 0, 0 for (fname, retcode, stdout, stderr, _) in results: if not retcode: successes += 1 print(Colors.OKGREEN + " import successful" + Colors.NO_COLOR) logging.debug(" import successful") else: # non-zero retcode failures += 1 if verbose: print(Colors.FAIL + " import failed" + Colors.NO_COLOR) logging.debug(" import failed") logging.warning(stderr) else: print(Colors.FAIL + " import failed (error in log)" + Colors.NO_COLOR) logging.debug(" import failed") logging.debug(stderr) print("translations: " + str(successes + failures)) print("successes...: " + str(successes)) print("failures....: " + str(failures)) return failures == 0
import os.path import re import toml from common import ( config as c, Colors, get_cmd_or_die, invoke, invoke_quietly, pb, setup_logging, ) git = get_cmd_or_die('git') cargo = get_cmd_or_die('cargo') python3 = get_cmd_or_die('python3') # These crates should be sorted in reverse dependency order. CRATES = [ c.MACROS_CRATE_DIR, # Not packaging cross-checking crates for now. # c.XCHECK_CONFIG_CRATE_DIR, # c.XCHECK_BACKEND_DYNAMIC_DLSYM_CRATE_DIR, # c.XCHECK_RUNTIME_CRATE_DIR, # c.XCHECK_DERIVE_CRATE_DIR, # c.XCHECK_PLUGIN_CRATE_DIR, c.AST_BUILDER_CRATE_DIR, c.AST_EXPORTER_CRATE_DIR,
#!/usr/bin/env python3 import argparse import errno import logging import os import multiprocessing import re import sys from common import (config as c, pb, Colors, die, get_cmd_or_die, invoke, regex, setup_logging, transpile, on_mac) cargo = get_cmd_or_die('cargo') git = get_cmd_or_die('git') intercept_build = get_cmd_or_die('intercept_build') make = get_cmd_or_die('make') python = get_cmd_or_die('python') rustc = get_cmd_or_die('rustc') NUM_JOBS = multiprocessing.cpu_count() EXAMPLES = [ 'genann', 'grabc', 'libxml2', 'lil', 'snudown', 'tmux', 'urlparser', 'xzoom' ] def build_path(path, new: str, is_dir: bool) -> str: err_msg = "`{}` does not exist in {}".format(new, path)
from typing import List import plumbum as pb from common import ( config as c, Colors, get_cmd_or_die, get_rust_toolchain_libpath, get_host_triplet, setup_logging, die, ) # Tools we will need rustfmt = get_cmd_or_die("rustfmt") diff = get_cmd_or_die("diff") def get_testcases(directory: str) -> List[str]: """ Find the test cases in a directory """ testcases = [] scriptnam = "run.sh" for root, subdirs, files in os.walk(directory): if scriptnam in files: testcases.append(os.path.join(root, scriptnam))
LUA_ARCHIVE = "lua-5.3.2.tar.gz" LUA_SRC = LUA_ARCHIVE.replace(".tar.gz", "") LUA_SRC = os.path.join(c.DEPS_DIR, LUA_SRC) RUBY_URL = "https://cache.ruby-lang.org/pub/ruby/2.4/ruby-2.4.1.tar.gz" RUBY_ARCHIVE = os.path.basename(RUBY_URL) RUBY_SRC = RUBY_ARCHIVE.replace(".tar.gz", "") RUBY_SRC = os.path.join(c.DEPS_DIR, RUBY_SRC) JSON_C_URL = "https://s3.amazonaws.com/" + \ "json-c_releases/releases/json-c-0.13.1.tar.gz" JSON_C_ARCHIVE = os.path.basename(JSON_C_URL) JSON_C_SRC = JSON_C_ARCHIVE.replace(".tar.gz", "") JSON_C_SRC = os.path.join(c.DEPS_DIR, JSON_C_SRC) TAR = get_cmd_or_die("tar") SED = get_cmd_or_die("sed") MAKE = get_cmd_or_die("make") CMAKE = get_cmd_or_die("cmake") BEAR = get_cmd_or_die("bear") JOBS = "-j2" # main updates jobs based on args minimal_snippet = """ \ int main() { return 0; } """ hello_world_snippet = """ \ #include <stdio.h> int main() { printf("Hello, World!\\n"); return 0;
def build_libfakechecks(): make = get_cmd_or_die("make") if not os.path.isfile(os.path.join(c.LIBFAKECHECKS_DIR, "libfakechecks.so")): with pb.local.cwd(c.LIBFAKECHECKS_DIR): invoke(make, "all")