def main(): # Used Environment variables: # # PKG_NODE_ROOT # Absolute path of a working directory for this script. # It checks out LLVM and Clang in "$PKG_NODE_ROOT/llvm", # builds it in "$PKG_NODE_ROOT/build", and installs it to # "$PKG_NODE_ROOT/libclang" # # CLANG_BRANCH # "Branch" identifier for the resulting package name # # cfg # Configuration containing of platform and bitness information # like "linux-g++-Rhel7.2-x64", "mac-clang-10.11-x64", # "win-MinGW5.3.0-Windows10-x64", "win-MinGW5.3.0-Windows10-x86", # "win-msvc2015-Windows10-x64", "win-msvc2015-Windows10-x86" # # GENERATE_INSTRUMENTED_BINARIES # Set this to 1 if you want to build MinGW libraries with information # suitable for creating profile optimized builds # # PACKAGE_STORAGE_SERVER_USER # PACKAGE_STORAGE_SERVER # PACKAGE_STORAGE_SERVER_BASE_DIR # CLANG_UPLOAD_SERVER_PATH # Define a remote path where to upload the resulting package # "PACKAGE_STORAGE_SERVER_USER@PACKAGE_STORAGE_SERVER:PACKAGE_STORAGE_SERVER_BASE_DIR/CLANG_UPLOAD_SERVER_PATH" # # LLVM_REVISION # Git revision, branch or tag for LLVM/Clang check out bldinstallercommon.init_common_module(os.path.dirname(os.path.realpath(__file__))) base_path = os.path.join(os.environ['PKG_NODE_ROOT']) branch = os.environ['CLANG_BRANCH'] src_path = os.path.join(base_path, 'llvm/llvm') build_path = os.path.join(base_path, 'build') install_path = os.path.join(base_path, 'libclang') bitness = 64 if '64' in os.environ['cfg'] else 32 toolchain = os.environ['cfg'].split('-')[1].lower() environment = build_environment(toolchain, bitness) result_file_path = os.path.join(base_path, 'libclang-' + branch + '-' + os.environ['CLANG_PLATFORM'] + '.7z') remote_path = (get_pkg_value("PACKAGE_STORAGE_SERVER_USER") + '@' + get_pkg_value("PACKAGE_STORAGE_SERVER") + ':' + os.environ['PACKAGE_STORAGE_SERVER_BASE_DIR'] + '/' + os.environ['CLANG_UPLOAD_SERVER_PATH']) get_clang(base_path, os.environ['LLVM_REVISION']) # TODO: put args in some struct to improve readability, add error checks build_clang(toolchain, src_path, build_path, install_path, bitness, environment, build_type='Release') check_clang(toolchain, build_path, environment) package_clang(install_path, result_file_path) upload_clang(result_file_path, remote_path)
def sign_windows_executable(file_path: str, signing_server: str, signing_pass: str, timestamp: str): signTools = ["signtool32.exe", "keys.pfx", "capicom.dll"] signToolsTempDir = r'C:\Utils\sign_tools_temp' for item in signTools: dst = os.path.join(signToolsTempDir, item) curl_cmd_args = [ 'curl', "--fail", "-L", "--retry", "5", "--retry-delay", "30", "-o", dst, '--create-dirs', get_pkg_value("SIGN_TOOLS_ADDR") + item ] subprocess.check_call(curl_cmd_args) cmd_args = [ os.path.join(signToolsTempDir, 'signtool32.exe'), 'sign', '/v', '/du', signing_server, '/p', signing_pass ] cmd_args += [ '/tr', timestamp, '/f', os.path.join(signToolsTempDir, 'keys.pfx'), '/td', "sha256", '/fd', "sha256", file_path ] log_entry = cmd_args[:] log_entry[4] = "****" log_entry[6] = "****" log.info("Calling: {0}".format(' '.join(log_entry))) subprocess.check_call(cmd_args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) shutil.rmtree(signToolsTempDir) log.info(f"Successfully signed: {file_path}")
def check_arguments(callerArguments): if bldinstallercommon.is_mac_platform(): if callerArguments.keychain_unlock_script: # inherited by child processes os.environ["SIGNING_IDENTITY"] = get_pkg_value("SIGNING_IDENTITY") os.environ["SIGNING_FLAGS"] = get_pkg_value("SIGNING_FLAGS") if not os.environ.get('SIGNING_IDENTITY') or not os.environ.get( 'SIGNING_FLAGS'): print( 'error: Environment variable(s) SIGNING_IDENTITY/SIGNING_FLAGS not set' ) sys.exit(1) if not os.path.lexists( callerArguments.qt5path) and not callerArguments.qt_modules: parser.print_help() print(( "error: You have to pass the --qt-module argument if the {0} does not exist" + os.linesep + os.linesep).format(callerArguments.qt5path)) sys.exit(1)
def sign_mac_app(app_path: str, signing_identity: str) -> None: assert app_path.endswith( ".app"), f"Not a valid path to .app bundle: {app_path}" # we need to unlock the keychain first unlock_script = "/Users/qt/unlock-keychain.sh" subprocess.check_call([unlock_script]) # "-o runtime" is required for notarization cmd_args = [ 'codesign', '-o', 'runtime', '--verbose=3', '-r', get_pkg_value("SIGNING_FLAGS"), '-s', signing_identity, app_path ] subprocess.check_call(cmd_args) log.info(f"Successfully signed: {app_path}")
parser = argparse.ArgumentParser( prog="Helper script to build a lib against qtbase artifact.") parser.add_argument("--qtpkg", dest="qtpkg", type=str, default=os.getenv("QT_PKG_URL"), help="URL pointing to pre-built Qt bin package.") parser.add_argument("--src-path", dest="src_path", type=str, default=os.getenv("SRC_PATH"), help="Path to sources") parser.add_argument("--remote-server", dest="remote_server", type=str, default=get_pkg_value("PACKAGE_STORAGE_SERVER"), help="Output server for build artifacts") parser.add_argument("--username", dest="username", type=str, default=get_pkg_value("PACKAGE_STORAGE_SERVER_USER"), help="Username for the output server") parser.add_argument("--remote-base-path", dest="remote_base_path", type=str, default=os.getenv("PACKAGE_STORAGE_SERVER_BASE_DIR"), help="Base path for output") parser.add_argument("--project-name", dest="project_name", type=str, default=os.getenv("PROJECT_NAME"),
if __name__ == "__main__": parser = argparse.ArgumentParser( prog= "Helper script to sign macOS .app bundle and create .dmg from it or sign a Windows .exe" ) subparsers = parser.add_subparsers(dest='command') app_parser = subparsers.add_parser("mac") exe_parser = subparsers.add_parser("win") app_parser.add_argument("--file", dest="file_path", required=True, help="Full path to .app file") app_parser.add_argument("--signing-identity", default=get_pkg_value("SIGNING_IDENTITY")) exe_parser.add_argument("--file", dest="file_path", required=True, help="Full path to .exe file") exe_parser.add_argument("--signing-server", required=False, default=get_pkg_value("SIGNING_SERVER")) exe_parser.add_argument("--signing-pass", required=False, default=get_pkg_value("SIGNING_PASSWORD")) exe_parser.add_argument("--timestamp", required=False, default="http://timestamp.digicert.com")
def main(): # Used Environment variables: # # PKG_NODE_ROOT # Absolute path of a working directory for this script. # It checks out LLVM and Clang in "$PKG_NODE_ROOT/llvm", # builds it in "$PKG_NODE_ROOT/build", and installs it to # "$PKG_NODE_ROOT/libclang" # # CLANG_BRANCH # "Branch" identifier for the resulting package name # # cfg # Configuration containing of platform and bitness information # like "linux-g++-Rhel7.2-x64", "mac-clang-10.11-x64", # "win-MinGW5.3.0-Windows10-x64", "win-MinGW5.3.0-Windows10-x86", # "win-msvc2015-Windows10-x64", "win-msvc2015-Windows10-x86" # # PACKAGE_STORAGE_SERVER_USER # PACKAGE_STORAGE_SERVER # PACKAGE_STORAGE_SERVER_BASE_DIR # CLANG_UPLOAD_SERVER_PATH # Define a remote path where to upload the resulting package # "PACKAGE_STORAGE_SERVER_USER@PACKAGE_STORAGE_SERVER:PACKAGE_STORAGE_SERVER_BASE_DIR/CLANG_UPLOAD_SERVER_PATH" # # LLVM_REPOSITORY_URL # URL to the remote llvm-project repository. # # LLVM_REVISION # Git revision, branch or tag for LLVM/Clang check out # # CLAZY_REPOSITORY_URL # URL to the remote clazy repository # # CLAZY_REVISION # Git revision, branch or tag for clazy check out bldinstallercommon.init_common_module( os.path.dirname(os.path.realpath(__file__))) base_path = os.path.join(os.environ['PKG_NODE_ROOT']) branch = os.environ['CLANG_BRANCH'] src_path = os.path.join(base_path, 'llvm/llvm') build_path = os.path.join(base_path, 'build') install_path = os.path.join(base_path, 'libclang') bitness = 64 if '64' in os.environ['cfg'] else 32 toolchain = os.environ['cfg'].split('-')[1].lower() environment = build_environment(toolchain, bitness) profile_data_path = os.path.join(base_path, 'profile_data') remote_path = (get_pkg_value("PACKAGE_STORAGE_SERVER_USER") + '@' + get_pkg_value("PACKAGE_STORAGE_SERVER") + ':' + os.environ['PACKAGE_STORAGE_SERVER_BASE_DIR'] + '/' + os.environ['CLANG_UPLOAD_SERVER_PATH']) ### Get, build and install LLVM/Clang get_clang(base_path, os.environ['LLVM_REPOSITORY_URL'], os.environ['LLVM_REVISION']) # TODO: put args in some struct to improve readability, add error checks build_clang(toolchain, src_path, build_path, install_path, profile_data_path, False, bitness, environment, build_type='Release') check_clang(toolchain, build_path, environment) if is_mingw_toolchain(toolchain): # We need to build libclang three times. # First time as a regular build, which would be used by a Qt Creator build to link to libclang/llvm. # Second time a PGO build, which would be trained with Qt Creator itself # Third time will use the training data collected and produce the optimized output if os.path.exists(profile_data_path): shutil.rmtree(profile_data_path) os.makedirs(profile_data_path) # Update the regular build, so that we can see the differences result_file_path = os.path.join( base_path, 'libclang-' + branch + '-' + os.environ['CLANG_PLATFORM'] + '-regular.7z') package_clang(install_path, result_file_path) upload_clang(result_file_path, remote_path) build_path_training = os.path.join(base_path, 'build-training') install_path_training = os.path.join(base_path, 'libclang-training') build_clang(toolchain, src_path, build_path_training, install_path_training, profile_data_path, True, bitness, environment, build_type='Release') mingw_training(base_path, os.path.join(base_path, 'qt-creator'), environment, bitness) build_clang(toolchain, src_path, build_path_training, install_path, profile_data_path, False, bitness, environment, build_type='Release') ### Get, build and install clazy git_clone_and_checkout(base_path, os.environ['CLAZY_REPOSITORY_URL'], 'clazy', os.environ['CLAZY_REVISION']) build_clazy(toolchain, os.path.join(base_path, 'clazy'), os.path.join(base_path, 'clazy-build'), install_path, bitness, environment) ### Package and upload result_file_path = os.path.join( base_path, 'libclang-' + branch + '-' + os.environ['CLANG_PLATFORM'] + '.7z') package_clang(install_path, result_file_path) upload_clang(result_file_path, remote_path)
def mingw_training(base_path, qtcreator_path, environment, bitness): # Checkout qt-creator, download libclang for build, qt installer and DebugView git_clone_and_checkout(base_path, 'git://code.qt.io/qt-creator/qt-creator.git', qtcreator_path, training_qtcreator_version()) # Set up paths script_dir = os.path.dirname(os.path.realpath(__file__)) debugview_dir = os.path.join(base_path, 'debugview') cmake_dir = os.path.join(base_path, 'cmake') creator_build_dir = os.path.join(base_path, 'qtcreator_build') creator_install_dir = os.path.join(base_path, 'qtcreator_install') creator_settings_dir = os.path.join(base_path, 'qtc-settings') creator_logs_dir = os.path.join(base_path, 'logs') training_dir = os.path.join(script_dir, 'libclang_training') qt_dir = os.path.join(base_path, 'qt') qt_mingw_dir = os.path.join(base_path, 'qt_mingw') # Create some paths os.makedirs(creator_settings_dir) os.makedirs(creator_logs_dir) pkg_server = get_pkg_value("PACKAGE_STORAGE_SERVER") # Install Qt qt_modules = [ 'qtbase', 'qtdeclarative', 'qtgraphicaleffects', 'qtimageformats', 'qtlocation', 'qtquickcontrols', 'qtquickcontrols2', 'qtscript', 'qtsvg', 'qttools', 'qttranslations', 'qtxmlpatterns' ] qt_base_url = 'http://' + pkg_server + '/packages/jenkins/archive/qt/' \ + training_qt_version() + '/' + training_qt_long_version() + '/latest' msvc_year_ver = msvc_year_version() if bitness == 64: qt_mingw_postfix = '-Windows-Windows_10-Mingw-Windows-Windows_10-X86_64.7z' qt_postfix = '-Windows-Windows_10-' + msvc_year_ver + '-Windows-Windows_10-X86_64.7z' else: qt_mingw_postfix = '-Windows-Windows_7-Mingw-Windows-Windows_7-X86.7z' qt_postfix = '-Windows-Windows_10-' + msvc_year_ver + '-Windows-Windows_10-X86.7z' qt_module_urls = [ qt_base_url + '/' + module + '/' + module + qt_postfix for module in qt_modules ] qt_mingw_module_urls = [ qt_base_url + '/' + module + '/' + module + qt_mingw_postfix for module in qt_modules ] qt_temp = os.path.join(base_path, 'qt_download') qt_mingw_temp = os.path.join(base_path, 'qt_download_mingw') download_packages_work = threadedwork.ThreadedWork("get and extract Qt") download_packages_work.addTaskObject( bldinstallercommon.create_qt_download_task(qt_module_urls, qt_dir, qt_temp, None)) download_packages_work.addTaskObject( bldinstallercommon.create_qt_download_task(qt_mingw_module_urls, qt_mingw_dir, qt_mingw_temp, None)) download_packages_work.addTaskObject( bldinstallercommon.create_download_extract_task( 'https://download.sysinternals.com/files/DebugView.zip', debugview_dir, base_path, None)) # Install CMake cmake_arch_suffix = 'win64-x64' if bitness == 64 else 'win32-x86' cmake_base_url = 'http://' + pkg_server + '/packages/jenkins/cmake/' \ + cmake_version() + '/cmake-' + cmake_version() + '-' + cmake_arch_suffix + '.zip' download_packages_work.addTaskObject( bldinstallercommon.create_download_extract_task( cmake_base_url, cmake_dir, base_path, None)) download_packages_work.run() # Build QtCreator with installed libclang and qt # WITH_TESTS is required for QtCreator to support running .batch files cmake_command = os.path.join( cmake_dir, 'cmake-' + cmake_version() + '-' + cmake_arch_suffix, 'bin', 'cmake') qtc_cmake = [ cmake_command, '-GNinja', '-DCMAKE_BUILD_TYPE=Release', '-DWITH_TESTS=ON', '-DBUILD_QBS=OFF', '-DBUILD_PLUGINS_BY_DEFAULT=OFF', '-DBUILD_EXECUTABLES_BY_DEFAULT=OFF', '-DBUILD_PLUGIN_CORE=ON', '-DBUILD_PLUGIN_TEXTEDITOR=ON', '-DBUILD_PLUGIN_PROJECTEXPLORER=ON', '-DBUILD_PLUGIN_CPPTOOLS=ON', '-DBUILD_PLUGIN_CPPEDITOR=ON', '-DBUILD_PLUGIN_QMAKEPROJECTMANAGER=ON', '-DBUILD_PLUGIN_CLANGCODEMODEL=ON', '-DBUILD_PLUGIN_CLANGTOOLS=ON', '-DBUILD_PLUGIN_DEBUGGER=ON', '-DBUILD_PLUGIN_DESIGNER=ON', '-DBUILD_PLUGIN_QTSUPPORT=ON', '-DBUILD_PLUGIN_RESOURCEEDITOR=ON', '-DBUILD_EXECUTABLE_QTCREATOR=ON', '-DBUILD_EXECUTABLE_ECHO=ON', '-DBUILD_EXECUTABLE_CLANGBACKEND=ON', '-DCMAKE_PREFIX_PATH=' + qt_mingw_dir + ';' + os.path.join(base_path, 'libclang'), '-S' + qtcreator_path, '-B' + creator_build_dir ] bld_utils.runCommand(qtc_cmake, creator_build_dir, None, environment) bld_utils.runCommand([cmake_command, '--build', creator_build_dir], creator_build_dir, None, environment) bld_utils.runCommand([ cmake_command, '--install', creator_build_dir, '--prefix', creator_install_dir ], creator_build_dir, None, environment) bld_utils.runCommand([ cmake_command, '--install', creator_build_dir, '--prefix', creator_install_dir, '--component', 'Dependencies' ], creator_build_dir, None, environment) # Remove the regular libclang.dll which got deployed via 'Dependencies' qtcreator install target os.remove(os.path.join(creator_install_dir, 'bin', 'libclang.dll')) # Train mingw libclang library with build QtCreator # First time open the project, then close it. This will generate initial settings and .user files. Second time do the actual training. for batchFile in ['qtc.openProject.batch', 'qtc.fileTextEditorCpp.batch']: bld_utils.runCommand([ os.path.join(training_dir, 'runBatchFiles.bat'), msvc_version(), 'x64' if bitness == 64 else 'x86', batchFile ], base_path, callerArguments=None, init_environment=None, onlyErrorCaseOutput=False, expectedExitCodes=[0, 1])
args.dmg)) await embedNotarization(args) if __name__ == "__main__": parser = argparse.ArgumentParser( prog="Helper script to notarize given macOS disk image (.dmg)") parser.add_argument("--dmg", dest="dmg", required=True, type=str, help=".dmg file") parser.add_argument("--user", dest="user", type=str, default=get_pkg_value("AC_USERNAME"), help="App Store Connect Username") parser.add_argument("--passwd", dest="passwd", type=str, default=get_pkg_value("AC_PASSWORD"), help="App Store Connect Password") parser.add_argument("--bundle-id", dest="bundle_id", default=strftime('%Y-%m-%d-%H-%M-%S', gmtime()), type=str, help="Give unique id for this bundle") parser.add_argument("--timeout", dest="timeout", type=int, default=60 * 60 * 3,
"[email protected] " "--staging-server-root=/base/path/for/online/repositories " "--config=<path to top level config .ini file> " "--task-filter=linux,x64,common --task-filter=linux,x64,opensource " "--artifacts-share-url=<http(s)://some.server.com/base/path/for/artifacts> " "--license=opensource " "--repo-domain=qtsdkrepository " "--build-repositories " "--update-staging " "--update-production " "--rta=<RTA trigger base URL> " "--sync-s3=<S3 bucket address> ") parser.add_argument("--ifw-tools", dest="ifw_tools", type=str, default=os.getenv("IFW_TOOLS"), help="Archive which contains the ifw tools") parser.add_argument("--staging-server", dest="staging_server", type=str, default=get_pkg_value("STAGING_SERVER_LOGIN"), help="Staging server") parser.add_argument("--staging-server-root", dest="staging_server_root", type=str, default=os.getenv("STAGING_SERVER_ROOT"), help="Online repositories root directory") parser.add_argument("--config", dest="config", type=str, default=os.getenv("RELEASE_DESCRIPTION_FILE"), help="Path to top level release config file") parser.add_argument("--task-filter", dest="task_filters", action='append', help="Task include filters per section name in the --config file to match with " "the section name, e.g. 'offline', 'repository', ...") parser.add_argument("--artifacts-share-url", dest="artifact_share_url", type=str, default=os.getenv("ARTIFACTS_SHARE_URL"), help="Root URL for artifacts") parser.add_argument("--license", dest="license", type=str, choices=["enterprise", "opensource"], default=os.getenv("LICENSE"), help="enterprise/opensource") parser.add_argument("--repo-domain", dest="repo_domain", type=str, choices=["qtsdkrepository", "marketplace"],