def main():
if sys.platform == "linux":
host_platform = "linux64"
default_target_triple = "x86_64-unknown-linux-gnu"
elif sys.platform == "darwin":
host_platform = "macos"
machine = platform.machine()
if machine == "arm64":
default_target_triple = "aarch64-apple-darwin"
elif machine == "x86_64":
default_target_triple = "x86_64-apple-darwin"
else:
raise Exception("unhandled macOS machine value: %s" % machine)
else:
print("unsupport build platform: %s" % sys.platform)
return 1
parser = argparse.ArgumentParser()
parser.add_argument(
"--target-triple",
default=default_target_triple,
choices=supported_targets(TARGETS_CONFIG),
help="Target host triple to build for",
)
parser.add_argument(
"--optimizations",
choices={"debug", "noopt", "pgo", "lto", "pgo+lto"},
default="noopt",
help="Optimizations to apply when compiling Python",
)
parser.add_argument(
"--python",
choices={"cpython-3.8", "cpython-3.9", "cpython-3.10"},
default="cpython-3.9",
help="Python distribution to build",
)
parser.add_argument(
"--break-on-failure",
action="store_true",
help="Enter a Python debugger if an error occurs",
)
parser.add_argument(
"--no-docker",
action="store_true",
default=True if sys.platform == "darwin" else False,
help="Disable building in Docker",
)
parser.add_argument(
"--serial",
action="store_true",
help="Build packages serially, without parallelism",
)
parser.add_argument(
"--make-target",
choices={"default", "toolchain"},
default="default",
help="The make target to evaluate",
)
args = parser.parse_args()
target_triple = args.target_triple
settings = get_target_settings(TARGETS_CONFIG, target_triple)
supported_pythons = {
"cpython-%s" % p
for p in settings["pythons_supported"]
}
if args.python not in supported_pythons:
print("%s only supports following Pythons: %s" %
(target_triple, ", ".join(supported_pythons)))
return 1
musl = "musl" in target_triple
env = dict(os.environ)
env["PYBUILD_HOST_PLATFORM"] = host_platform
env["PYBUILD_TARGET_TRIPLE"] = target_triple
env["PYBUILD_OPTIMIZATIONS"] = args.optimizations
if musl:
env["PYBUILD_MUSL"] = "1"
if args.break_on_failure:
env["PYBUILD_BREAK_ON_FAILURE"] = "1"
if args.no_docker:
env["PYBUILD_NO_DOCKER"] = "1"
entry = DOWNLOADS[args.python]
env["PYBUILD_PYTHON_VERSION"] = entry["version"]
env["PYBUILD_PYTHON_MAJOR_VERSION"] = ".".join(
entry["version"].split(".")[0:2])
if "PYBUILD_RELEASE_TAG" in os.environ:
release_tag = os.environ["PYBUILD_RELEASE_TAG"]
else:
release_tag = release_tag_from_git()
archive_components = [
"cpython-%s" % entry["version"],
target_triple,
args.optimizations,
]
build_basename = "-".join(archive_components) + ".tar"
dist_basename = "-".join(archive_components + [release_tag])
# We run make with static parallelism no greater than the machine's CPU count
# because we can get some speedup from parallel operations. But we also don't
# share a make job server with each build. So if we didn't limit the
# parallelism we could easily oversaturate the CPU. Higher levels of
# parallelism don't result in meaningful build speedups because tk/tix has
# a long, serial dependency chain that can't be built in parallel.
parallelism = min(1 if args.serial else 4, multiprocessing.cpu_count())
subprocess.run(["make", "-j%d" % parallelism, args.make_target],
env=env,
check=True)
DIST.mkdir(exist_ok=True)
if args.make_target == "default":
compress_python_archive(BUILD / build_basename, DIST, dist_basename)
def main():
BUILD.mkdir(exist_ok=True)
DOWNLOADS_PATH.mkdir(exist_ok=True)
(BUILD / "logs").mkdir(exist_ok=True)
if os.environ.get("PYBUILD_NO_DOCKER"):
client = None
else:
try:
client = docker.from_env()
client.ping()
except Exception as e:
print("unable to connect to Docker: %s" % e)
return 1
parser = argparse.ArgumentParser()
parser.add_argument(
"--host-platform", required=True, help="Platform we are building from"
)
parser.add_argument(
"--target-triple",
required=True,
help="Host triple that we are building Python for",
)
parser.add_argument(
"--optimizations",
choices={"debug", "noopt", "pgo", "lto", "pgo+lto"},
required=True,
help="Optimization profile to use",
)
parser.add_argument(
"--toolchain",
action="store_true",
help="Indicates we are building a toolchain artifact",
)
parser.add_argument(
"--dest-archive", required=True, help="Path to archive that we are producing"
)
parser.add_argument("--docker-image", help="Docker image to use for building")
parser.add_argument("action")
args = parser.parse_args()
action = args.action
target_triple = args.target_triple
host_platform = args.host_platform
optimizations = args.optimizations
dest_archive = pathlib.Path(args.dest_archive)
docker_image = args.docker_image
settings = get_target_settings(TARGETS_CONFIG, target_triple)
if args.action == "makefiles":
log_name = "makefiles"
elif args.action.startswith("image-"):
log_name = "image-%s" % action
elif args.toolchain:
log_name = "%s-%s" % (action, host_platform)
else:
entry = DOWNLOADS[action]
log_name = "%s-%s-%s-%s" % (
action,
entry["version"],
target_triple,
optimizations,
)
log_path = BUILD / "logs" / ("build.%s.log" % log_name)
with log_path.open("wb") as log_fh:
set_logger(action, log_fh)
if action == "makefiles":
write_triples_makefiles(get_targets(TARGETS_CONFIG), BUILD)
write_package_versions(BUILD / "versions")
elif action.startswith("image-"):
build_docker_image(client, ROOT, BUILD, action[6:])
elif action == "binutils":
build_binutils(client, get_image(client, ROOT, BUILD, "gcc"), host_platform)
elif action == "clang":
build_clang(
client,
get_image(client, ROOT, BUILD, "clang"),
host_platform=host_platform,
)
elif action == "gcc":
build_gcc(client, get_image(client, ROOT, BUILD, "gcc"), host_platform)
elif action == "musl":
build_musl(client, get_image(client, ROOT, BUILD, "gcc"), host_platform)
elif action == "libedit":
build_libedit(
settings,
client,
get_image(client, ROOT, BUILD, docker_image),
host_platform=host_platform,
target_triple=target_triple,
optimizations=optimizations,
dest_archive=dest_archive,
)
elif action == "readline":
build_readline(
settings,
client,
get_image(client, ROOT, BUILD, docker_image),
host_platform=host_platform,
target_triple=target_triple,
optimizations=optimizations,
dest_archive=dest_archive,
)
elif action in (
"bdb",
"bzip2",
"gdbm",
"gettext",
"inputproto",
"kbproto",
"libffi",
"libpthread-stubs",
"libressl",
"ncurses",
"openssl",
"sqlite",
"tcl",
"uuid",
"x11-util-macros",
"xextproto",
"xorgproto",
"xproto",
"xtrans",
"xz",
"zlib",
):
simple_build(
settings,
client,
get_image(client, ROOT, BUILD, docker_image),
action,
host_platform=host_platform,
target_triple=target_triple,
optimizations=optimizations,
dest_archive=dest_archive,
)
elif action == "libX11":
simple_build(
settings,
client,
get_image(client, ROOT, BUILD, docker_image),
action,
host_platform=host_platform,
target_triple=target_triple,
optimizations=optimizations,
dest_archive=dest_archive,
extra_archives={
"inputproto",
"kbproto",
"libpthread-stubs",
"libXau",
"libxcb",
"x11-util-macros",
"xextproto",
"xorgproto",
"xproto",
"xtrans",
},
)
elif action == "libXau":
simple_build(
settings,
client,
get_image(client, ROOT, BUILD, docker_image),
action,
host_platform=host_platform,
target_triple=target_triple,
optimizations=optimizations,
dest_archive=dest_archive,
extra_archives={"x11-util-macros", "xproto"},
)
elif action == "xcb-proto":
simple_build(
settings,
client,
get_image(client, ROOT, BUILD, docker_image),
action,
host_platform=host_platform,
target_triple=target_triple,
optimizations=optimizations,
dest_archive=dest_archive,
)
elif action == "libxcb":
simple_build(
settings,
client,
get_image(client, ROOT, BUILD, docker_image),
action,
host_platform=host_platform,
target_triple=target_triple,
optimizations=optimizations,
dest_archive=dest_archive,
extra_archives={"libpthread-stubs", "libXau", "xcb-proto", "xproto"},
)
elif action == "tix":
build_tix(
settings,
client,
get_image(client, ROOT, BUILD, docker_image),
host_platform=host_platform,
target_triple=target_triple,
optimizations=optimizations,
dest_archive=dest_archive,
)
elif action == "tk":
extra_archives = {"tcl"}
if host_platform != "macos":
extra_archives |= {
"libX11",
"libXau",
"libxcb",
"xcb-proto",
"xorgproto",
}
simple_build(
settings,
client,
get_image(client, ROOT, BUILD, docker_image),
action,
host_platform=host_platform,
target_triple=target_triple,
optimizations=optimizations,
dest_archive=dest_archive,
extra_archives=extra_archives,
)
elif action in ("cpython-3.8", "cpython-3.9", "cpython-3.10"):
build_cpython(
settings,
client,
get_image(client, ROOT, BUILD, docker_image),
host_platform=host_platform,
target_triple=target_triple,
optimizations=optimizations,
dest_archive=dest_archive,
libressl="PYBUILD_LIBRESSL" in os.environ,
version=action.split("-")[1],
)
else:
print("unknown build action: %s" % action)
return 1
def main():
if sys.platform == "linux":
host_platform = "linux64"
default_target_triple = "x86_64-unknown-linux-gnu"
elif sys.platform == "darwin":
host_platform = "macos"
machine = platform.machine()
if machine == "arm64":
default_target_triple = "aarch64-apple-darwin"
elif machine == "x86_64":
default_target_triple = "x86_64-apple-darwin"
else:
raise Exception("unhandled macOS machine value: %s" % machine)
else:
print("unsupport build platform: %s" % sys.platform)
return 1
parser = argparse.ArgumentParser()
parser.add_argument(
"--target-triple",
default=default_target_triple,
choices=supported_targets(TARGETS_CONFIG),
help="Target host triple to build for",
)
parser.add_argument(
"--optimizations",
choices={"debug", "noopt", "pgo", "lto", "pgo+lto"},
default="noopt",
help="Optimizations to apply when compiling Python",
)
parser.add_argument("--libressl",
action="store_true",
help="Build LibreSSL instead of OpenSSL")
parser.add_argument(
"--python",
choices={"cpython-3.8", "cpython-3.9", "cpython-3.10"},
default="cpython-3.9",
help="Python distribution to build",
)
parser.add_argument(
"--break-on-failure",
action="store_true",
help="Enter a Python debugger if an error occurs",
)
parser.add_argument(
"--no-docker",
action="store_true",
default=True if sys.platform == "darwin" else False,
help="Disable building in Docker",
)
parser.add_argument(
"--skip-toolchain",
action="store_true",
help=
"Skip building the toolchain (requires a tar file in expected location)",
)
parser.add_argument(
"--make-target",
choices={"default", "toolchain"},
default="default",
help="The make target to evaluate",
)
args = parser.parse_args()
target_triple = args.target_triple
settings = get_target_settings(TARGETS_CONFIG, target_triple)
if args.python not in settings["pythons_supported"]:
print("%s only supports following Pythons: %s" %
(target_triple, ", ".join(settings["pythons_supported"])))
return 1
musl = "musl" in target_triple
env = dict(os.environ)
env["PYBUILD_HOST_PLATFORM"] = host_platform
env["PYBUILD_TARGET_TRIPLE"] = target_triple
env["PYBUILD_OPTIMIZATIONS"] = args.optimizations
if args.libressl or musl:
env["PYBUILD_LIBRESSL"] = "1"
if musl:
env["PYBUILD_MUSL"] = "1"
if args.break_on_failure:
env["PYBUILD_BREAK_ON_FAILURE"] = "1"
if args.no_docker:
env["PYBUILD_NO_DOCKER"] = "1"
if args.skip_toolchain:
env["PYBUILD_SKIP_TOOLCHAIN"] = "1"
entry = DOWNLOADS[args.python]
env["PYBUILD_PYTHON_VERSION"] = entry["version"]
env["PYBUILD_PYTHON_MAJOR_VERSION"] = ".".join(
entry["version"].split(".")[0:2])
if "PYBUILD_RELEASE_TAG" in os.environ:
release_tag = os.environ["PYBUILD_RELEASE_TAG"]
else:
release_tag = release_tag_from_git()
archive_components = [
"cpython-%s" % entry["version"],
target_triple,
args.optimizations,
]
build_basename = "-".join(archive_components) + ".tar"
dist_basename = "-".join(archive_components + [release_tag])
subprocess.run(["make", args.make_target], env=env, check=True)
DIST.mkdir(exist_ok=True)
if args.make_target == "default":
compress_python_archive(BUILD / build_basename, DIST, dist_basename)
def add_target_env(env, build_platform, target_triple, build_env):
add_env_common(env)
settings = get_target_settings(TARGETS_CONFIG, target_triple)
env["HOST_CC"] = settings["host_cc"]
env["CC"] = settings["target_cc"]
env["PYBUILD_PLATFORM"] = build_platform
env["TOOLS_PATH"] = build_env.tools_path
extra_target_cflags = list(settings.get("target_cflags", []))
extra_target_ldflags = list(settings.get("target_ldflags", []))
extra_host_cflags = []
extra_host_ldflags = []
if build_platform == "linux64":
env["BUILD_TRIPLE"] = "x86_64-unknown-linux-gnu"
# TODO should the musl target be normalized?
if target_triple == "x86_64-unknown-linux-musl":
env["TARGET_TRIPLE"] = "x86_64-unknown-linux-gnu"
else:
env["TARGET_TRIPLE"] = target_triple
if build_platform == "macos":
machine = platform.machine()
if machine == "arm64":
env["BUILD_TRIPLE"] = "aarch64-apple-darwin"
elif machine == "x86_64":
env["BUILD_TRIPLE"] = "x86_64-apple-darwin"
else:
raise Exception("unhandled macOS machine value: %s" % machine)
# Sniff out the Apple SDK minimum deployment target from cflags and
# export in its own variable. This is used by CPython's configure, as
# it doesn't sniff the cflag.
for flag in extra_target_cflags:
m = re.search("-version-min=(.*)$", flag)
if m:
env["APPLE_MIN_DEPLOYMENT_TARGET"] = m.group(1)
break
else:
raise Exception("could not find minimum Apple SDK version in cflags")
sdk_platform = settings["apple_sdk_platform"]
env["APPLE_SDK_PLATFORM"] = sdk_platform
env["TARGET_TRIPLE"] = target_triple
# We don't have build isolation on macOS. We nerf PATH to prevent
# non-system (e.g. Homebrew) executables from being used.
env["PATH"] = "/usr/bin:/bin"
if "APPLE_SDK_PATH" in os.environ:
sdk_path = os.environ["APPLE_SDK_PATH"]
else:
# macOS SDK has historically been in /usr courtesy of an
# installer provided by Xcode. But with Catalina, the files
# are now typically in
# /Applications/Xcode.app/Contents/Developer/Platforms/.
# The proper way to resolve this path is with xcrun, which
# will give us the headers that Xcode is configured to use.
res = subprocess.run(
["xcrun", "--sdk", sdk_platform, "--show-sdk-path"],
check=True,
capture_output=True,
encoding="utf-8",
)
sdk_path = res.stdout.strip()
if not os.path.exists(sdk_path):
raise Exception("macOS SDK path %s does not exist" % sdk_path)
# Grab the version from the SDK so we can put it in PYTHON.json.
sdk_settings_path = pathlib.Path(sdk_path) / "SDKSettings.json"
with sdk_settings_path.open("rb") as fh:
sdk_settings = json.load(fh)
env["APPLE_SDK_VERSION"] = sdk_settings["Version"]
env["APPLE_SDK_CANONICAL_NAME"] = sdk_settings["CanonicalName"]
extra_target_cflags.extend(["-isysroot", sdk_path])
extra_target_ldflags.extend(["-isysroot", sdk_path])
# The host SDK may be for a different platform from the target SDK.
# Resolve that separately.
if "APPLE_HOST_SDK_PATH" in os.environ:
host_sdk_path = os.environ["APPLE_HOST_SDK_PATH"]
else:
host_sdk_path = subprocess.run(
["xcrun", "--show-sdk-path"],
check=True,
capture_output=True,
encoding="utf-8",
).stdout.strip()
if not os.path.exists(host_sdk_path):
raise Exception("macOS host SDK path %s does not exist" % host_sdk_path)
extra_host_cflags.extend(["-isysroot", host_sdk_path])
extra_host_ldflags.extend(["-isysroot", host_sdk_path])
env["EXTRA_HOST_CFLAGS"] = " ".join(extra_host_cflags)
env["EXTRA_HOST_LDFLAGS"] = " ".join(extra_host_ldflags)
env["EXTRA_TARGET_CFLAGS"] = " ".join(extra_target_cflags)
env["EXTRA_TARGET_LDFLAGS"] = " ".join(extra_target_ldflags)