def main(): # Install protobuf on linux/mac if its not installed already if not utils.is_command_installed('protoc'): if utils.is_linux_os(): # sudo apt install protobuf-compiler utils.run_command(['apt', 'install', '-y', 'protobuf-compiler'], as_root=True) elif utils.is_mac_os(): # brew install protobuf utils.run_command(['brew', 'install', 'protobuf']) # Install go on linux/mac if its not installed already if not utils.is_command_installed('go'): if utils.is_linux_os(): # sudo apt install -y golang utils.run_command(['apt', 'install', '-y', 'golang'], as_root=True) elif utils.is_mac_os(): # brew install protobuf utils.run_command(['brew', 'install', 'go']) # Install openssl on linux/mac if its not installed already if not utils.is_command_installed('go'): if utils.is_linux_os(): # sudo apt install -y openssl utils.run_command(['apt', 'install', '-y', 'openssl'], as_root=True) elif utils.is_mac_os(): # brew install protobuf utils.run_command(['brew', 'install', 'openssl']) # Install ccache on linux/mac if its not installed already if not utils.is_command_installed('ccache'): if utils.is_linux_os(): # sudo apt install ccache utils.run_command(['apt', 'install', '-y', 'ccache'], as_root=True) elif utils.is_mac_os(): # brew install ccache utils.run_command(['brew', 'install', 'ccache']) # Install clang-format on linux/mac if its not installed already if not utils.is_command_installed('clang-format'): if utils.is_linux_os(): # sudo apt install clang-format utils.run_command(['apt', 'install', '-y', 'clang-format'], as_root=True) elif utils.is_mac_os(): # brew install protobuf utils.run_command(['brew', 'install', 'clang-format']) # Install required python dependencies. # On Catalina, python2 in installed as default python. # Example command: # python3 -m pip install -r external/pip_requirements.txt --user utils.run_command([ 'python3' if utils.is_command_installed('python3') else 'python', '-m', 'pip', 'install', '-r', 'external/pip_requirements.txt', '--user' ])
def install_x86_support_libraries(gha_build=False): """Install support libraries needed to build x86 on x86_64 hosts. Args: gha_build: Pass in True if running on a GitHub runner; this will activate workarounds that might be undesirable on a personal system (e.g. downgrading Ubuntu packages). """ if utils.is_linux_os(): packages = ['gcc-multilib', 'g++-multilib', 'libglib2.0-dev:i386', 'libsecret-1-dev:i386', 'libpthread-stubs0-dev:i386', 'libssl-dev:i386'] if gha_build: # Workaround for GitHub runners, which have an incompatibility between the # 64-bit and 32-bit versions of the Ubuntu package libpcre2-8-0. Downgrade # the installed 64-bit version of the library to get around this issue. # This will presumably be fixed in a future Ubuntu update. (If you remove # it, remove the workaround further down this function as well.) packages = ['--allow-downgrades'] + packages + ['libpcre2-8-0=10.34-7'] # First check if these packages exist on the machine already devnull = open(os.devnull, "w") process = subprocess.run(["dpkg", "-s"] + packages, stdout=devnull, stderr=subprocess.STDOUT) devnull.close() if process.returncode != 0: # This implies not all of the required packages are already installed on user's machine # Install them. utils.run_command(['dpkg', '--add-architecture', 'i386'], as_root=True, check=True) utils.run_command(['apt', 'update'], as_root=True, check=True) utils.run_command(['apt', 'install', '-V', '-y'] + packages, as_root=True, check=True) if gha_build: # One more workaround: downgrading libpcre2-8-0 above may have uninstalled # libsecret, which is required for the Linux build. Force it to be # reinstalled, but do it as a separate command to ensure that held # packages aren't modified. (Once the workaround above is removed, this can # be removed as well.) # Note: "-f" = "fix" - let apt do what it needs to do to fix dependencies. utils.run_command(['apt', 'install', '-f', '-V', '-y', 'libsecret-1-dev'], as_root=True, check=True)
def inspect_arch_linux(lib): """Find the CPU architecture for the specified library. Args: lib (str): Path to a library file on disk. Returns: (str): Filtered output from objdump tool. """ if not utils.is_linux_os(): print( "ERROR: inspect_arch_linux function can only be called on linux. Exiting " ) sys.exit(1) objdump = subprocess.Popen(['objdump', '-f', lib], stdout=subprocess.PIPE) grep = subprocess.Popen(['grep', '-B', '1', 'architecture'], stdin=objdump.stdout, stdout=subprocess.PIPE) output = grep.communicate()[0] if not output: return return output.decode('utf-8')
def main(): args = parse_cmdline_args() # Ensure that the submodules are initialized and updated # Example: vcpkg is a submodule (external/vcpkg) if not args.disable_vcpkg: utils.run_command(['git', 'submodule', 'init'], check=True) utils.run_command(['git', 'submodule', 'update'], check=True) # To build x86 on x86_64 linux hosts, we also need x86 support libraries if args.arch == 'x86' and utils.is_linux_os(): install_x86_support_libraries(args.gha_build) # Install C++ dependencies using vcpkg if not args.disable_vcpkg: # Install C++ dependencies using vcpkg install_cpp_dependencies_with_vcpkg(args.arch, args.msvc_runtime_library, cleanup=True, use_openssl=args.use_openssl) if args.vcpkg_step_only: print("Exiting without building the Firebase C++ SDK as just vcpkg step was requested.") return # CMake configure cmake_configure(args.build_dir, args.arch, args.msvc_runtime_library, args.linux_abi, args.build_tests, args.config, args.target_format, args.use_openssl, args.disable_vcpkg, args.gha_build, args.verbose) # CMake build # cmake --build build -j 8 cmd = ['cmake', '--build', args.build_dir, '-j', str(os.cpu_count()), '--config', args.config] if args.target: # Example: cmake --build build -j 8 --target firebase_app firebase_auth cmd.append('--target') cmd.extend(args.target) utils.run_command(cmd)
def cmake_configure(build_dir, arch, msvc_runtime_library='static', linux_abi='legacy', build_tests=True, config=None, target_format=None, use_openssl=False, disable_vcpkg=False, gha_build=False, verbose=False): """ CMake configure. If you are seeing problems when running this multiple times, make sure to clean/delete previous build directory. Args: build_dir (str): Output build directory. arch (str): Platform Architecture (example: 'x64', 'x86', 'arm64'). msvc_runtime_library (str): Runtime library for MSVC (eg: 'static', 'dynamic'). linux_abi (str): Linux ABI (eg: 'legacy', 'c++11'). build_tests (bool): Build cpp unit tests. config (str): Release/Debug config. If its not specified, cmake's default is used (most likely Debug). target_format (str): If specified, build for this targetformat ('frameworks' or 'libraries'). use_openssl (bool) : Use prebuilt OpenSSL library instead of using boringssl downloaded and built during the cmake configure step. disable_vcpkg (bool): If True, skip vcpkg and just use CMake for deps. gha_build (bool): If True, this build will be marked as having been built from GitHub, which is useful for metrics tracking. verbose (bool): If True, enable verbose mode in the CMake file. """ cmd = ['cmake', '-S', '.', '-B', build_dir] # If generator is not specifed, default for platform is used by cmake, else # use the specified value if config: cmd.append('-DCMAKE_BUILD_TYPE={0}'.format(config)) if build_tests: cmd.append('-DFIREBASE_CPP_BUILD_TESTS=ON') cmd.append('-DFIREBASE_FORCE_FAKE_SECURE_STORAGE=ON') else: # workaround, absl doesn't build without tests enabled cmd.append('-DBUILD_TESTING=off') if not disable_vcpkg: if utils.is_linux_os() and arch == 'x86': # Use a separate cmake toolchain for cross compiling linux x86 builds vcpkg_toolchain_file_path = os.path.join(os.getcwd(), 'external', 'vcpkg', 'scripts', 'buildsystems', 'linux_32.cmake') elif utils.is_mac_os() and arch == 'arm64': vcpkg_toolchain_file_path = os.path.join(os.getcwd(), 'external', 'vcpkg', 'scripts', 'buildsystems', 'macos_arm64.cmake') else: vcpkg_toolchain_file_path = os.path.join(os.getcwd(), 'external', 'vcpkg', 'scripts', 'buildsystems', 'vcpkg.cmake') cmd.append('-DCMAKE_TOOLCHAIN_FILE={0}'.format(vcpkg_toolchain_file_path)) vcpkg_triplet = utils.get_vcpkg_triplet(arch, msvc_runtime_library) cmd.append('-DVCPKG_TARGET_TRIPLET={0}'.format(vcpkg_triplet)) if utils.is_windows_os(): # If building for x86, we should supply -A Win32 to cmake configure # Its a good habit to specify for x64 too as the default might be different # on different windows machines. cmd.append('-A') cmd.append('Win32') if arch == 'x86' else cmd.append('x64') # Use our special cmake flag to specify /MD vs /MT if msvc_runtime_library == "static": cmd.append('-DMSVC_RUNTIME_LIBRARY_STATIC=ON') if utils.is_mac_os(): if (arch == 'arm64'): cmd.append('-DCMAKE_OSX_ARCHITECTURES=arm64') else: cmd.append('-DCMAKE_OSX_ARCHITECTURES=x86_64') if utils.is_linux_os() and linux_abi == 'c++11': cmd.append('-DFIREBASE_LINUX_USE_CXX11_ABI=TRUE') if (target_format): cmd.append('-DFIREBASE_XCODE_TARGET_FORMAT={0}'.format(target_format)) if not use_openssl: cmd.append('-DFIREBASE_USE_BORINGSSL=ON') # When building from GitHub Actions, this should always be set. if gha_build: cmd.append('-DFIREBASE_GITHUB_ACTION_BUILD=ON') # Print out every command while building. if verbose: cmd.append('-DCMAKE_VERBOSE_MAKEFILE=1') utils.run_command(cmd)
def main(): args = parse_cmdline_args() summary_headers = ['Library', 'Architecture'] # Setup platform specific functions if utils.is_linux_os(): arch_pattern = get_arch_re_pattern_linux() inspect_arch_function = inspect_arch_linux summarize_arch_function = functools.partial(summarize_arch_linux, re_pattern=arch_pattern) elif utils.is_mac_os(): arch_pattern = get_arch_re_pattern_mac() inspect_arch_function = inspect_arch_mac summarize_arch_function = functools.partial(summarize_arch_mac, re_pattern=arch_pattern) elif utils.is_windows_os(): arch_pattern = get_arch_re_pattern_windows() dumpbin = get_or_create_dumpbin_exe_path() inspect_arch_function = functools.partial(inspect_arch_windows, dumpbin_exe_path=dumpbin) summarize_arch_function = functools.partial(summarize_arch_windows, re_pattern=arch_pattern) summary_headers.append('MSVC_Runtime') msvc_runtime_library_pattern = get_msvc_runtime_library_re_pattern() else: raise ValueError( "Unsupported desktop OS. Can be either Mac, Linux or Windows.") # We technically dont support full blown regex or glob filters but still cover # the simplest and most frequently used symbol *, just incase. Avoiding # regex for speed and convenience (users can easily forget quotes around # and that expands to # all files in current directory). libs = get_libraries_to_inspect(args.libraries, args.library_filter.replace('*', '')) all_libs_info = [] for lib in libs: libname = os.path.basename(lib) if not args.print_full_paths else lib libinfo = [libname] try: verbose_data = inspect_arch_function(lib) if args.verbose: print("Inspecting architecture: {0}".format(lib)) print(verbose_data) print() arch = summarize_arch_function(verbose_data) libinfo.append(arch) except: libinfo.append('N.A') if utils.is_windows_os(): try: verbose_data = inspect_msvc_runtime_library( lib, dumpbin_exe_path=dumpbin) if args.verbose: print("Inspecting msvc runtime library: {0}".format(lib)) print(verbose_data) print() runtime_library = summarize_msvc_runtime_library( verbose_data, re_pattern=msvc_runtime_library_pattern) libinfo.append(runtime_library) except: libinfo.append('N.A') all_libs_info.append(libinfo) # Sort libraries info based on library names. all_libs_info.sort(key=lambda x: x[0]) print_summary_table(summary_headers, all_libs_info)
def main(argv): if len(argv) > 1: raise app.UsageError("Too many command-line arguments.") platforms = FLAGS.platforms testapps = FLAGS.testapps sdk_dir = _fix_path(FLAGS.packaged_sdk or FLAGS.repo_dir) root_output_dir = _fix_path(FLAGS.output_directory) repo_dir = _fix_path(FLAGS.repo_dir) update_pod_repo = FLAGS.update_pod_repo if FLAGS.add_timestamp: timestamp = datetime.datetime.now().strftime("%Y_%m_%d-%H_%M_%S") else: timestamp = "" if FLAGS.short_output_paths: output_dir = os.path.join(root_output_dir, "ta") else: output_dir = os.path.join(root_output_dir, "testapps" + timestamp) config = config_reader.read_config() xcframework_dir = os.path.join(sdk_dir, "xcframeworks") xcframework_exist = os.path.isdir(xcframework_dir) if not xcframework_exist: if _IOS in platforms: _build_xcframework_from_repo(repo_dir, "ios", testapps, config) if _TVOS in platforms: _build_xcframework_from_repo(repo_dir, "tvos", testapps, config) if update_pod_repo and (_IOS in platforms or _TVOS in platforms): _run(["pod", "repo", "update"]) cmake_flags = _get_desktop_compiler_flags(FLAGS.compiler, config.compilers) # VCPKG is used to install dependencies for the desktop SDK. # Building from source requires building the underlying SDK libraries, # so we need to use VCPKG as well. if _DESKTOP in platforms and not FLAGS.packaged_sdk: vcpkg_arch = FLAGS.arch installer = os.path.join(repo_dir, "scripts", "gha", "build_desktop.py") _run([sys.executable, installer, "--vcpkg_step_only", "--arch", vcpkg_arch]) toolchain_file = os.path.join( repo_dir, "external", "vcpkg", "scripts", "buildsystems", "vcpkg.cmake") if utils.is_mac_os() and FLAGS.arch == "arm64": toolchain_file = os.path.join( repo_dir, "external", "vcpkg", "scripts", "buildsystems", "macos_arm64.cmake") if utils.is_linux_os() and FLAGS.arch == "x86": toolchain_file = os.path.join( repo_dir, "external", "vcpkg", "scripts", "buildsystems", "linux_32.cmake") cmake_flags.extend(( "-DCMAKE_TOOLCHAIN_FILE=%s" % toolchain_file, "-DVCPKG_TARGET_TRIPLET=%s" % utils.get_vcpkg_triplet(arch=vcpkg_arch) )) if FLAGS.cmake_flag: cmake_flags.extend(FLAGS.cmake_flag) failures = [] for testapp in testapps: api_config = config.get_api(testapp) if FLAGS.repo_dir and not FLAGS.packaged_sdk and api_config.internal_testapp_path: testapp_dirs = [api_config.internal_testapp_path] else: testapp_dirs = [api_config.testapp_path] for testapp_dir in testapp_dirs: logging.info("BEGIN building for %s: %s", testapp, testapp_dir) failures += _build( testapp=testapp, platforms=platforms, api_config=config.get_api(testapp), testapp_dir=testapp_dir, output_dir=output_dir, sdk_dir=sdk_dir, xcframework_exist=xcframework_exist, repo_dir=repo_dir, ios_sdk=FLAGS.ios_sdk, tvos_sdk=FLAGS.tvos_sdk, cmake_flags=cmake_flags, short_output_paths=FLAGS.short_output_paths) logging.info("END building for %s", testapp) _collect_integration_tests(testapps, root_output_dir, output_dir, FLAGS.artifact_name) _summarize_results(testapps, platforms, failures, root_output_dir, FLAGS.artifact_name) return 1 if failures else 0