def main():
# Used Environment variables:
#
# PKG_NODE_ROOT
# Absolute path of a working directory for this script.
# It checks out LLVM and Clang in "$PKG_NODE_ROOT/llvm",
# builds it in "$PKG_NODE_ROOT/build", and installs it to
# "$PKG_NODE_ROOT/libclang"
#
# CLANG_BRANCH
# "Branch" identifier for the resulting package name
#
# cfg
# Configuration containing of platform and bitness information
# like "linux-g++-Rhel7.2-x64", "mac-clang-10.11-x64",
# "win-MinGW5.3.0-Windows10-x64", "win-MinGW5.3.0-Windows10-x86",
# "win-msvc2015-Windows10-x64", "win-msvc2015-Windows10-x86"
#
# GENERATE_INSTRUMENTED_BINARIES
# Set this to 1 if you want to build MinGW libraries with information
# suitable for creating profile optimized builds
#
# PACKAGE_STORAGE_SERVER_USER
# PACKAGE_STORAGE_SERVER
# PACKAGE_STORAGE_SERVER_BASE_DIR
# CLANG_UPLOAD_SERVER_PATH
# Define a remote path where to upload the resulting package
# "PACKAGE_STORAGE_SERVER_USER@PACKAGE_STORAGE_SERVER:PACKAGE_STORAGE_SERVER_BASE_DIR/CLANG_UPLOAD_SERVER_PATH"
#
# LLVM_REVISION
# Git revision, branch or tag for LLVM/Clang check out
bldinstallercommon.init_common_module(os.path.dirname(os.path.realpath(__file__)))
base_path = os.path.join(os.environ['PKG_NODE_ROOT'])
branch = os.environ['CLANG_BRANCH']
src_path = os.path.join(base_path, 'llvm/llvm')
build_path = os.path.join(base_path, 'build')
install_path = os.path.join(base_path, 'libclang')
bitness = 64 if '64' in os.environ['cfg'] else 32
toolchain = os.environ['cfg'].split('-')[1].lower()
environment = build_environment(toolchain, bitness)
result_file_path = os.path.join(base_path, 'libclang-' + branch + '-' + os.environ['CLANG_PLATFORM'] + '.7z')
remote_path = (get_pkg_value("PACKAGE_STORAGE_SERVER_USER") + '@' + get_pkg_value("PACKAGE_STORAGE_SERVER") + ':'
+ os.environ['PACKAGE_STORAGE_SERVER_BASE_DIR'] + '/' + os.environ['CLANG_UPLOAD_SERVER_PATH'])
get_clang(base_path, os.environ['LLVM_REVISION'])
# TODO: put args in some struct to improve readability, add error checks
build_clang(toolchain, src_path, build_path, install_path, bitness, environment, build_type='Release')
check_clang(toolchain, build_path, environment)
package_clang(install_path, result_file_path)
upload_clang(result_file_path, remote_path)
def sign_windows_executable(file_path: str, signing_server: str,
signing_pass: str, timestamp: str):
signTools = ["signtool32.exe", "keys.pfx", "capicom.dll"]
signToolsTempDir = r'C:\Utils\sign_tools_temp'
for item in signTools:
dst = os.path.join(signToolsTempDir, item)
curl_cmd_args = [
'curl', "--fail", "-L", "--retry", "5", "--retry-delay", "30",
"-o", dst, '--create-dirs',
get_pkg_value("SIGN_TOOLS_ADDR") + item
]
subprocess.check_call(curl_cmd_args)
cmd_args = [
os.path.join(signToolsTempDir, 'signtool32.exe'), 'sign', '/v', '/du',
signing_server, '/p', signing_pass
]
cmd_args += [
'/tr', timestamp, '/f',
os.path.join(signToolsTempDir, 'keys.pfx'), '/td', "sha256", '/fd',
"sha256", file_path
]
log_entry = cmd_args[:]
log_entry[4] = "****"
log_entry[6] = "****"
log.info("Calling: {0}".format(' '.join(log_entry)))
subprocess.check_call(cmd_args,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
shutil.rmtree(signToolsTempDir)
log.info(f"Successfully signed: {file_path}")
def check_arguments(callerArguments):
if bldinstallercommon.is_mac_platform():
if callerArguments.keychain_unlock_script:
# inherited by child processes
os.environ["SIGNING_IDENTITY"] = get_pkg_value("SIGNING_IDENTITY")
os.environ["SIGNING_FLAGS"] = get_pkg_value("SIGNING_FLAGS")
if not os.environ.get('SIGNING_IDENTITY') or not os.environ.get(
'SIGNING_FLAGS'):
print(
'error: Environment variable(s) SIGNING_IDENTITY/SIGNING_FLAGS not set'
)
sys.exit(1)
if not os.path.lexists(
callerArguments.qt5path) and not callerArguments.qt_modules:
parser.print_help()
print((
"error: You have to pass the --qt-module argument if the {0} does not exist"
+ os.linesep + os.linesep).format(callerArguments.qt5path))
sys.exit(1)
def sign_mac_app(app_path: str, signing_identity: str) -> None:
assert app_path.endswith(
".app"), f"Not a valid path to .app bundle: {app_path}"
# we need to unlock the keychain first
unlock_script = "/Users/qt/unlock-keychain.sh"
subprocess.check_call([unlock_script])
# "-o runtime" is required for notarization
cmd_args = [
'codesign', '-o', 'runtime', '--verbose=3', '-r',
get_pkg_value("SIGNING_FLAGS"), '-s', signing_identity, app_path
]
subprocess.check_call(cmd_args)
log.info(f"Successfully signed: {app_path}")
parser = argparse.ArgumentParser(
prog="Helper script to build a lib against qtbase artifact.")
parser.add_argument("--qtpkg",
dest="qtpkg",
type=str,
default=os.getenv("QT_PKG_URL"),
help="URL pointing to pre-built Qt bin package.")
parser.add_argument("--src-path",
dest="src_path",
type=str,
default=os.getenv("SRC_PATH"),
help="Path to sources")
parser.add_argument("--remote-server",
dest="remote_server",
type=str,
default=get_pkg_value("PACKAGE_STORAGE_SERVER"),
help="Output server for build artifacts")
parser.add_argument("--username",
dest="username",
type=str,
default=get_pkg_value("PACKAGE_STORAGE_SERVER_USER"),
help="Username for the output server")
parser.add_argument("--remote-base-path",
dest="remote_base_path",
type=str,
default=os.getenv("PACKAGE_STORAGE_SERVER_BASE_DIR"),
help="Base path for output")
parser.add_argument("--project-name",
dest="project_name",
type=str,
default=os.getenv("PROJECT_NAME"),
if __name__ == "__main__":
parser = argparse.ArgumentParser(
prog=
"Helper script to sign macOS .app bundle and create .dmg from it or sign a Windows .exe"
)
subparsers = parser.add_subparsers(dest='command')
app_parser = subparsers.add_parser("mac")
exe_parser = subparsers.add_parser("win")
app_parser.add_argument("--file",
dest="file_path",
required=True,
help="Full path to .app file")
app_parser.add_argument("--signing-identity",
default=get_pkg_value("SIGNING_IDENTITY"))
exe_parser.add_argument("--file",
dest="file_path",
required=True,
help="Full path to .exe file")
exe_parser.add_argument("--signing-server",
required=False,
default=get_pkg_value("SIGNING_SERVER"))
exe_parser.add_argument("--signing-pass",
required=False,
default=get_pkg_value("SIGNING_PASSWORD"))
exe_parser.add_argument("--timestamp",
required=False,
default="http://timestamp.digicert.com")
def main():
# Used Environment variables:
#
# PKG_NODE_ROOT
# Absolute path of a working directory for this script.
# It checks out LLVM and Clang in "$PKG_NODE_ROOT/llvm",
# builds it in "$PKG_NODE_ROOT/build", and installs it to
# "$PKG_NODE_ROOT/libclang"
#
# CLANG_BRANCH
# "Branch" identifier for the resulting package name
#
# cfg
# Configuration containing of platform and bitness information
# like "linux-g++-Rhel7.2-x64", "mac-clang-10.11-x64",
# "win-MinGW5.3.0-Windows10-x64", "win-MinGW5.3.0-Windows10-x86",
# "win-msvc2015-Windows10-x64", "win-msvc2015-Windows10-x86"
#
# PACKAGE_STORAGE_SERVER_USER
# PACKAGE_STORAGE_SERVER
# PACKAGE_STORAGE_SERVER_BASE_DIR
# CLANG_UPLOAD_SERVER_PATH
# Define a remote path where to upload the resulting package
# "PACKAGE_STORAGE_SERVER_USER@PACKAGE_STORAGE_SERVER:PACKAGE_STORAGE_SERVER_BASE_DIR/CLANG_UPLOAD_SERVER_PATH"
#
# LLVM_REPOSITORY_URL
# URL to the remote llvm-project repository.
#
# LLVM_REVISION
# Git revision, branch or tag for LLVM/Clang check out
#
# CLAZY_REPOSITORY_URL
# URL to the remote clazy repository
#
# CLAZY_REVISION
# Git revision, branch or tag for clazy check out
bldinstallercommon.init_common_module(
os.path.dirname(os.path.realpath(__file__)))
base_path = os.path.join(os.environ['PKG_NODE_ROOT'])
branch = os.environ['CLANG_BRANCH']
src_path = os.path.join(base_path, 'llvm/llvm')
build_path = os.path.join(base_path, 'build')
install_path = os.path.join(base_path, 'libclang')
bitness = 64 if '64' in os.environ['cfg'] else 32
toolchain = os.environ['cfg'].split('-')[1].lower()
environment = build_environment(toolchain, bitness)
profile_data_path = os.path.join(base_path, 'profile_data')
remote_path = (get_pkg_value("PACKAGE_STORAGE_SERVER_USER") + '@' +
get_pkg_value("PACKAGE_STORAGE_SERVER") + ':' +
os.environ['PACKAGE_STORAGE_SERVER_BASE_DIR'] + '/' +
os.environ['CLANG_UPLOAD_SERVER_PATH'])
### Get, build and install LLVM/Clang
get_clang(base_path, os.environ['LLVM_REPOSITORY_URL'],
os.environ['LLVM_REVISION'])
# TODO: put args in some struct to improve readability, add error checks
build_clang(toolchain,
src_path,
build_path,
install_path,
profile_data_path,
False,
bitness,
environment,
build_type='Release')
check_clang(toolchain, build_path, environment)
if is_mingw_toolchain(toolchain):
# We need to build libclang three times.
# First time as a regular build, which would be used by a Qt Creator build to link to libclang/llvm.
# Second time a PGO build, which would be trained with Qt Creator itself
# Third time will use the training data collected and produce the optimized output
if os.path.exists(profile_data_path):
shutil.rmtree(profile_data_path)
os.makedirs(profile_data_path)
# Update the regular build, so that we can see the differences
result_file_path = os.path.join(
base_path, 'libclang-' + branch + '-' +
os.environ['CLANG_PLATFORM'] + '-regular.7z')
package_clang(install_path, result_file_path)
upload_clang(result_file_path, remote_path)
build_path_training = os.path.join(base_path, 'build-training')
install_path_training = os.path.join(base_path, 'libclang-training')
build_clang(toolchain,
src_path,
build_path_training,
install_path_training,
profile_data_path,
True,
bitness,
environment,
build_type='Release')
mingw_training(base_path, os.path.join(base_path, 'qt-creator'),
environment, bitness)
build_clang(toolchain,
src_path,
build_path_training,
install_path,
profile_data_path,
False,
bitness,
environment,
build_type='Release')
### Get, build and install clazy
git_clone_and_checkout(base_path, os.environ['CLAZY_REPOSITORY_URL'],
'clazy', os.environ['CLAZY_REVISION'])
build_clazy(toolchain, os.path.join(base_path, 'clazy'),
os.path.join(base_path, 'clazy-build'), install_path, bitness,
environment)
### Package and upload
result_file_path = os.path.join(
base_path,
'libclang-' + branch + '-' + os.environ['CLANG_PLATFORM'] + '.7z')
package_clang(install_path, result_file_path)
upload_clang(result_file_path, remote_path)
def mingw_training(base_path, qtcreator_path, environment, bitness):
# Checkout qt-creator, download libclang for build, qt installer and DebugView
git_clone_and_checkout(base_path,
'git://code.qt.io/qt-creator/qt-creator.git',
qtcreator_path, training_qtcreator_version())
# Set up paths
script_dir = os.path.dirname(os.path.realpath(__file__))
debugview_dir = os.path.join(base_path, 'debugview')
cmake_dir = os.path.join(base_path, 'cmake')
creator_build_dir = os.path.join(base_path, 'qtcreator_build')
creator_install_dir = os.path.join(base_path, 'qtcreator_install')
creator_settings_dir = os.path.join(base_path, 'qtc-settings')
creator_logs_dir = os.path.join(base_path, 'logs')
training_dir = os.path.join(script_dir, 'libclang_training')
qt_dir = os.path.join(base_path, 'qt')
qt_mingw_dir = os.path.join(base_path, 'qt_mingw')
# Create some paths
os.makedirs(creator_settings_dir)
os.makedirs(creator_logs_dir)
pkg_server = get_pkg_value("PACKAGE_STORAGE_SERVER")
# Install Qt
qt_modules = [
'qtbase', 'qtdeclarative', 'qtgraphicaleffects', 'qtimageformats',
'qtlocation', 'qtquickcontrols', 'qtquickcontrols2', 'qtscript',
'qtsvg', 'qttools', 'qttranslations', 'qtxmlpatterns'
]
qt_base_url = 'http://' + pkg_server + '/packages/jenkins/archive/qt/' \
+ training_qt_version() + '/' + training_qt_long_version() + '/latest'
msvc_year_ver = msvc_year_version()
if bitness == 64:
qt_mingw_postfix = '-Windows-Windows_10-Mingw-Windows-Windows_10-X86_64.7z'
qt_postfix = '-Windows-Windows_10-' + msvc_year_ver + '-Windows-Windows_10-X86_64.7z'
else:
qt_mingw_postfix = '-Windows-Windows_7-Mingw-Windows-Windows_7-X86.7z'
qt_postfix = '-Windows-Windows_10-' + msvc_year_ver + '-Windows-Windows_10-X86.7z'
qt_module_urls = [
qt_base_url + '/' + module + '/' + module + qt_postfix
for module in qt_modules
]
qt_mingw_module_urls = [
qt_base_url + '/' + module + '/' + module + qt_mingw_postfix
for module in qt_modules
]
qt_temp = os.path.join(base_path, 'qt_download')
qt_mingw_temp = os.path.join(base_path, 'qt_download_mingw')
download_packages_work = threadedwork.ThreadedWork("get and extract Qt")
download_packages_work.addTaskObject(
bldinstallercommon.create_qt_download_task(qt_module_urls, qt_dir,
qt_temp, None))
download_packages_work.addTaskObject(
bldinstallercommon.create_qt_download_task(qt_mingw_module_urls,
qt_mingw_dir, qt_mingw_temp,
None))
download_packages_work.addTaskObject(
bldinstallercommon.create_download_extract_task(
'https://download.sysinternals.com/files/DebugView.zip',
debugview_dir, base_path, None))
# Install CMake
cmake_arch_suffix = 'win64-x64' if bitness == 64 else 'win32-x86'
cmake_base_url = 'http://' + pkg_server + '/packages/jenkins/cmake/' \
+ cmake_version() + '/cmake-' + cmake_version() + '-' + cmake_arch_suffix + '.zip'
download_packages_work.addTaskObject(
bldinstallercommon.create_download_extract_task(
cmake_base_url, cmake_dir, base_path, None))
download_packages_work.run()
# Build QtCreator with installed libclang and qt
# WITH_TESTS is required for QtCreator to support running .batch files
cmake_command = os.path.join(
cmake_dir, 'cmake-' + cmake_version() + '-' + cmake_arch_suffix, 'bin',
'cmake')
qtc_cmake = [
cmake_command, '-GNinja', '-DCMAKE_BUILD_TYPE=Release',
'-DWITH_TESTS=ON', '-DBUILD_QBS=OFF', '-DBUILD_PLUGINS_BY_DEFAULT=OFF',
'-DBUILD_EXECUTABLES_BY_DEFAULT=OFF', '-DBUILD_PLUGIN_CORE=ON',
'-DBUILD_PLUGIN_TEXTEDITOR=ON', '-DBUILD_PLUGIN_PROJECTEXPLORER=ON',
'-DBUILD_PLUGIN_CPPTOOLS=ON', '-DBUILD_PLUGIN_CPPEDITOR=ON',
'-DBUILD_PLUGIN_QMAKEPROJECTMANAGER=ON',
'-DBUILD_PLUGIN_CLANGCODEMODEL=ON', '-DBUILD_PLUGIN_CLANGTOOLS=ON',
'-DBUILD_PLUGIN_DEBUGGER=ON', '-DBUILD_PLUGIN_DESIGNER=ON',
'-DBUILD_PLUGIN_QTSUPPORT=ON', '-DBUILD_PLUGIN_RESOURCEEDITOR=ON',
'-DBUILD_EXECUTABLE_QTCREATOR=ON', '-DBUILD_EXECUTABLE_ECHO=ON',
'-DBUILD_EXECUTABLE_CLANGBACKEND=ON', '-DCMAKE_PREFIX_PATH=' +
qt_mingw_dir + ';' + os.path.join(base_path, 'libclang'),
'-S' + qtcreator_path, '-B' + creator_build_dir
]
bld_utils.runCommand(qtc_cmake, creator_build_dir, None, environment)
bld_utils.runCommand([cmake_command, '--build', creator_build_dir],
creator_build_dir, None, environment)
bld_utils.runCommand([
cmake_command, '--install', creator_build_dir, '--prefix',
creator_install_dir
], creator_build_dir, None, environment)
bld_utils.runCommand([
cmake_command, '--install', creator_build_dir, '--prefix',
creator_install_dir, '--component', 'Dependencies'
], creator_build_dir, None, environment)
# Remove the regular libclang.dll which got deployed via 'Dependencies' qtcreator install target
os.remove(os.path.join(creator_install_dir, 'bin', 'libclang.dll'))
# Train mingw libclang library with build QtCreator
# First time open the project, then close it. This will generate initial settings and .user files. Second time do the actual training.
for batchFile in ['qtc.openProject.batch', 'qtc.fileTextEditorCpp.batch']:
bld_utils.runCommand([
os.path.join(training_dir, 'runBatchFiles.bat'),
msvc_version(), 'x64' if bitness == 64 else 'x86', batchFile
],
base_path,
callerArguments=None,
init_environment=None,
onlyErrorCaseOutput=False,
expectedExitCodes=[0, 1])
args.dmg))
await embedNotarization(args)
if __name__ == "__main__":
parser = argparse.ArgumentParser(
prog="Helper script to notarize given macOS disk image (.dmg)")
parser.add_argument("--dmg",
dest="dmg",
required=True,
type=str,
help=".dmg file")
parser.add_argument("--user",
dest="user",
type=str,
default=get_pkg_value("AC_USERNAME"),
help="App Store Connect Username")
parser.add_argument("--passwd",
dest="passwd",
type=str,
default=get_pkg_value("AC_PASSWORD"),
help="App Store Connect Password")
parser.add_argument("--bundle-id",
dest="bundle_id",
default=strftime('%Y-%m-%d-%H-%M-%S', gmtime()),
type=str,
help="Give unique id for this bundle")
parser.add_argument("--timeout",
dest="timeout",
type=int,
default=60 * 60 * 3,
"[email protected] " "--staging-server-root=/base/path/for/online/repositories " "--config=<path to top level config .ini file> " "--task-filter=linux,x64,common --task-filter=linux,x64,opensource " "--artifacts-share-url=<http(s)://some.server.com/base/path/for/artifacts> " "--license=opensource " "--repo-domain=qtsdkrepository " "--build-repositories " "--update-staging " "--update-production " "--rta=<RTA trigger base URL> " "--sync-s3=<S3 bucket address> ") parser.add_argument("--ifw-tools", dest="ifw_tools", type=str, default=os.getenv("IFW_TOOLS"), help="Archive which contains the ifw tools") parser.add_argument("--staging-server", dest="staging_server", type=str, default=get_pkg_value("STAGING_SERVER_LOGIN"), help="Staging server") parser.add_argument("--staging-server-root", dest="staging_server_root", type=str, default=os.getenv("STAGING_SERVER_ROOT"), help="Online repositories root directory") parser.add_argument("--config", dest="config", type=str, default=os.getenv("RELEASE_DESCRIPTION_FILE"), help="Path to top level release config file") parser.add_argument("--task-filter", dest="task_filters", action='append', help="Task include filters per section name in the --config file to match with " "the section name, e.g. 'offline', 'repository', ...") parser.add_argument("--artifacts-share-url", dest="artifact_share_url", type=str, default=os.getenv("ARTIFACTS_SHARE_URL"), help="Root URL for artifacts") parser.add_argument("--license", dest="license", type=str, choices=["enterprise", "opensource"], default=os.getenv("LICENSE"), help="enterprise/opensource") parser.add_argument("--repo-domain", dest="repo_domain", type=str, choices=["qtsdkrepository", "marketplace"],
