def download_source(path):
"""Download sources for the package whose abs dir is path.
If sources are not downloaded for some reason, a file called
/sources/$PKGNAME.log will contain more information about the
reason.
"""
abs_name = os.path.basename(path)
source_dir = os.path.join("/sources", abs_name)
shutil.copytree(path, source_dir)
recursive_chown(source_dir)
# makepkg must be run in the directory where the PKGBUILD is.
# However, we cannot use os.chdir because the cwd is process-wide,
# not thread-local, and we're using threading rather than
# multiprocessing. Therefore, pass a new cwd to the subprocess.
#
# We want to download and extract the source, but not build it.
# PKGBUILDs also have a prepare() function that gets executed
# after extraction but before building; we shouldn't run that
# function either, since it might need some of the packages in
# `makedepends` to be installed. We should avoid installing those,
# since we would need to lock otherwise (only one Pacman can be
# running at a time).
#
# We don't pass --syncdeps (so that dependencies don't get
# installed) and also pass --nodeps (so that makepkg doesn't error
# out when it notices that not all dependencies are installed),
# because we're not building anything. Anything that is required for
# downloading and source extraction should already be installed by
# the create_base_image/main.sh script.
#
# GPG and SHA take forever, and there's a serious risk that we fill
# up the container's process table with unreaped children if we do
# integrity checks; so don't bother.
cmd = ("sudo -u tuscan makepkg --noprepare --nobuild --nocheck"
" --nodeps --noarchive --skipinteg --nocolor --nosign")
proc = subprocess.Popen(cmd.split(),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd=source_dir)
try:
out, _ = proc.communicate(timeout=1800)
except subprocess.TimeoutExpired:
report_failure("sources", "timed out", abs_name)
return
output = codecs.decode(out, errors="replace")
if proc.returncode:
shutil.rmtree(source_dir)
report_failure("sources", output, abs_name)
def get_package_source_dir(args):
"""Get the sources for a package.
PKGBUILDs contain instructions for downloading sources and then
building them. We don't want to download the sources before every
build, so this function downloads sources and stores them in a
standard location so that they can be copied later rather than
re-downloaded.
If this function returns successfully, the abs directory for package
will have been copied to sources_directory, and the sources for that
package will have been downloaded into it.
"""
if not os.path.isdir(args.abs_dir):
die(Status.failure, "Could not find abs directory for dir '%s'" % args.abs_dir)
shutil.copytree(args.abs_dir, args.permanent_source_dir)
recursive_chown(args.permanent_source_dir)
os.chdir(args.permanent_source_dir)
# The --nobuild flag to makepkg causes it to download sources, but
# not build them.
command = (
"sudo -u tuscan makepkg --nobuild --syncdeps "
"--skipinteg --skippgpcheck --skipchecksums "
"--noconfirm --nocolor --log --noprogressbar "
"--nocheck --nodeps"
)
time = timestamp()
cp = subprocess.run(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
log("command", command, cp.stdout.splitlines(), time)
success = False
if cp.returncode:
shutil.rmtree(args.permanent_source_dir)
else:
success = True
return success
def copy_and_build(args):
try:
shutil.copytree(args.permanent_source_dir, args.build_dir)
except shutil.Error as e:
# e.args will be a list, containing a single list of 3-tuples.
# We are interested in the third item of each tuple.
errors = [err[2] for err in e.args[0]]
die(Status.failure, "No source directory in source volume: %s" % args.permanent_source_dir, output=errors)
recursive_chown(args.build_dir)
os.chdir(args.build_dir)
proc = subprocess.Popen(["/usr/bin/sloccount", "src"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, _ = proc.communicate()
output = codecs.decode(out, errors="replace")
if proc.returncode:
log("die", "SLOCCount failed", output.splitlines())
else:
log_sloc(output.splitlines())
# Add the --host option to invocations of ./configure
with open("PKGBUILD", encoding="utf-8") as f:
pkgbuild = f.read().splitlines()
pkgbuild = [re.sub(r"configure\s", "configure --host x86_64-unknown-linux ", line) for line in pkgbuild]
with open("PKGBUILD", "w", encoding="utf-8") as f:
f.write("\n".join(pkgbuild))
# The difference between this invocation and the one in
# get_package_source_dir() is the --noextract flag. Sources should
# already have been downloaded and extracted by
# get_package_source_dir(), so we just want to build them.
#
# Also, the invocation in get_package_source_dir has the --nodeps
# option, since we just wanted to download sources there. Here, we
# do want to install dependencies (from our local toolchain
# repository), so don't pass the --nodeps flag.
if args.env_vars == None:
args.env_vars = []
command_env = os.environ.copy()
for pair in args.env_vars:
var, val = pair.split("=")
command_env[var] = val
command = (
"sudo -u tuscan " + " ".join(args.env_vars) + " bear makepkg --noextract --syncdeps"
" --skipinteg --skippgpcheck --skipchecksums"
" --noconfirm --nocolor --log --noprogressbar"
" --nocheck"
)
time = timestamp()
proc = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=command_env)
stdout_data, _ = proc.communicate()
output = codecs.decode(stdout_data, errors="replace")
log("command", command, output.splitlines(), time)
# Pick up output left by bear
if os.path.exists("compile_commands.json"):
with open("compile_commands.json") as f:
bear_output = json.load(f)
log("bear", "bear", output=bear_output)
else:
log("die", "No bear output found in dir '%s'" % os.getcwd())
native_tools = {}
for native in glob("/tmp/tuscan-native-*"):
with open(native) as f:
tool = f.readlines()
if tool:
tool = tool[0].strip()
if tool not in native_tools:
native_tools[tool] = 0
native_tools[tool] += 1
if native_tools:
log("native_tools", "native_tools", native_tools)
return proc.returncode
def toolchain_specific_setup(args):
log("info", "Running android-specific setup")
if not os.path.isdir("/sysroot"):
os.mkdir("/sysroot")
recursive_chown("/sysroot")
# wget and curl output unsuitable progress bars even when not
# connected to a TTY. Turn them off.
with open("/etc/wgetrc", "a") as f:
print("verbose = off", file=f)
with open("/etc/.curlrc", "a") as f:
print("silent", file=f)
print("show-error", file=f)
log("info", "Downloading & unpacking NDK")
os.chdir("/home/tuscan")
setup_file = "/home/tuscan/ndk.bin"
cmd = ("wget -O %s"
" http://dl.google.com/android/ndk/android-"
"ndk-r10e-linux-x86_64.bin" % (setup_file))
run_cmd(cmd)
cmd = "chmod +x " + setup_file
run_cmd(cmd)
run_cmd(setup_file, output=False)
log("info", "Setting up toolchain")
cmd = ("/home/tuscan/android-ndk-r10e/build/tools/"
"make-standalone-toolchain.sh"
" --arch=arm --platform=android-21 "
" --install-dir=" + "/sysroot")
run_cmd(cmd)
cmd = "chown -R tuscan: " + "/sysroot"
run_cmd(cmd, as_root=True)
cmd = "chown -R tuscan: /home/tuscan/android-ndk-r10e"
run_cmd(cmd, as_root=True)
bindirs = [
"/sysroot/bin", "/sysroot/libexec/gcc/arm-linux-androideabi/4.8"
]
for d in bindirs:
for f in os.listdir(d):
f = os.path.join(d, f)
cmd = "chmod a+rx %s" % f
run_cmd(cmd, as_root=True)
for f in os.listdir("/sysroot"):
if os.path.isdir(os.path.join("/sysroot", f)):
shutil.copytree(os.path.join("/sysroot", f),
os.path.join("/toolchain_root", f))
elif os.path.isfile(os.path.join("/sysroot", f)):
shutil.copy(os.path.join("/sysroot", f), "/toolchain_root")
recursive_chown("/toolchain_root")
def main():
"""Install vanilla bootstrap packages from local mirror.
Installing all the bootstrap packages is a lengthy (and highly
disk-IO bound, thus serializing) procedure, so it's best to do it
only once. Instead of having each container running the make_package
stage installing the boostrap packages, we install the bootstrap
packages in this container and then base the make_package containers
on the image of this container.
"""
parser = get_argparser()
args = parser.parse_args()
# GPG takes time. Remove package signature checks.
lines = []
with open("/etc/pacman.conf") as f:
for line in f:
if re.search("SigLevel", line):
lines.append("SigLevel = Never")
else:
lines.append(line.strip())
with open("/etc/pacman.conf", "w") as f:
for line in lines:
print(line.strip(), file=f)
name_data_file = os.path.join(args.shared_directory,
"get_base_package_names", "latest",
"names.json")
with open(name_data_file) as f:
name_data = json.load(f)
bootstrap_packs = (name_data["base"] + name_data["base_devel"] +
name_data["tools"] + ["sloccount"])
cmd = "pacman -S --needed --noconfirm %s" % " ".join(set(bootstrap_packs))
proc = subprocess.Popen(cmd.split(),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, _ = proc.communicate()
out = codecs.decode(out, errors="replace")
if proc.returncode:
log("die", cmd, out.splitlines())
exit(1)
else:
log("command", cmd, out.splitlines())
# When building red, we need to supply it with a list of defines
# suitable for this toolchain. Construct those defines here and
# write out the PKGBUILD with those defines.
with open("/build/tool_redirect_rules.yaml") as f:
transforms = yaml.load(f)
log("info",
"Before rules %s" % yaml.dump(transforms, default_flow_style=False))
for tool in transforms["overwrite"]:
transforms["replacements"][tool] = tool
log("info",
"After rules %s" % yaml.dump(transforms, default_flow_style=False))
defines = []
for tool, replacement in transforms["replacements"].items():
# The tool & replacement will be written just like the name of
# the tool binary, e.g. "scan-view", "clang++", etc. These are
# not valid identifiers (because they contain - or +), so the
# libred cmake define variable will write them as SCAN_VIEW and
# CLANGPP. Do that transformation here, but leave the name of
# the original tool intact.
var_name = re.sub("-", "_", tool)
var_name = re.sub("\+\+", "pp", var_name)
var_name = var_name.upper()
path = os.path.join(transforms["bin-dir"],
"%s%s" % (transforms["prefix"], replacement))
defines.append('-DRED_%s="%s"' % (var_name, path))
log("info", "Redirecting %s to %s" % (var_name, path))
if transforms["bin-dir"]:
defines.append('-DRED_ENSURE_PATH="%s"' % transforms["bin-dir"])
jinja = jinja2.Environment(loader=jinja2.FileSystemLoader(["/build"]))
pkgbuild_temp = jinja.get_template("red-PKGBUILD")
pkgbuild = pkgbuild_temp.render(defines=(" ".join(defines)))
with open("/build/PKGBUILD", "w") as f:
f.write(pkgbuild)
log("info", "Generated PKGBUILD for red", output=pkgbuild.splitlines())
# Build and install red
with tempfile.TemporaryDirectory() as d:
red_tar = os.path.join(d, "red.tar.xz")
with tarfile.open(red_tar, "w:xz") as tar:
tar.add("/red", arcname="red")
shutil.copyfile("/build/PKGBUILD", os.path.join(d, "PKGBUILD"))
shutil.chown(d, user="******")
os.chdir(d)
cmd = "sudo -u tuscan makepkg --nocolor"
cp = subprocess.run(cmd.split(),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True)
if cp.returncode:
log("die", cmd, cp.stdout.splitlines())
exit(1)
else:
log("command", cmd, cp.stdout.splitlines())
package = glob("red*.pkg.tar.xz")
if not len(package) == 1:
log("die", "More than one package found", package)
exit(1)
cmd = "pacman -U --noconfirm %s" % package[0]
cp = subprocess.run(cmd.split(),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True)
if cp.returncode:
log("die", cmd, cp.stdout.splitlines())
exit(1)
else:
log("command", cmd, cp.stdout.splitlines())
if not os.path.isdir("/toolchain_root"):
log("die", "/toolchain_root is not mounted")
exit(1)
if os.listdir("/toolchain_root"):
log("info", ("Skipping toolchain-specific setup as "
"/toolchain_root contains files. Listing:"),
output=list(os.listdir("/toolchain_root")))
else:
log("info", ("/toolchain_root is empty, performing "
"toolchain-specific setup"),
output=list(os.listdir("/toolchain_root")))
setup.toolchain_specific_setup(args)
recursive_chown("/toolchain_root")
exit(0)
def copy_and_build(args):
try:
shutil.copytree(args.permanent_source_dir, args.build_dir)
except shutil.Error as e:
# e.args will be a list, containing a single list of 3-tuples.
# We are interested in the third item of each tuple.
errors = [err[2] for err in e.args[0]]
die(Status.failure,
"No source directory in source volume: %s" %
args.permanent_source_dir,
output=errors)
recursive_chown(args.build_dir)
os.chdir(args.build_dir)
# Add the --host option to invocations of ./configure
with open("PKGBUILD", encoding="utf-8", errors="ignore") as f:
pkgbuild = f.read().splitlines()
if args.toolchain == "android":
pkgbuild = [
re.sub(r"configure\s", ("configure --build=x86_64-unknown-linux "
"--host=arm-linux-androideabi "), line)
for line in pkgbuild
]
else:
pkgbuild = [
re.sub(r"configure\s", "configure --host=x86_64-unknown-linux ",
line) for line in pkgbuild
]
with open("PKGBUILD", "w", encoding="utf-8") as f:
f.write("\n".join(pkgbuild))
# This invocation of makepkg has the --noextract flag, because
# sources should already have been extracted during the creation of
# the base image (see stages/create_base_image/getpkgs.py). We still
# need to perform all other stages of package building, including
# the prepare() function that is called just before the build()
# function.
#
# The invocation also has the --syncdeps flag; this is fine, because
# anything that this package depends on should already have been
# built and its hybrid package will have been installed.
if args.env_vars == None:
args.env_vars = []
command_env = os.environ.copy()
for pair in args.env_vars:
var, val = pair.split("=")
command_env[var] = val
command = ("sudo -u tuscan " + " ".join(args.env_vars) +
" red makepkg --noextract --syncdeps"
" --skipinteg --skippgpcheck --skipchecksums"
" --noconfirm --nocolor --log --noprogressbar"
" --nocheck")
time = timestamp()
proc = subprocess.Popen(command.split(),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=command_env)
stdout_data, _ = proc.communicate()
output = codecs.decode(stdout_data, errors="replace")
log("command", command, output.splitlines(), time)
# Measure LOC
loc_proc = subprocess.Popen(
["/usr/bin/sloccount", "--addlang", "makefile", "src"],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, _ = loc_proc.communicate()
output = codecs.decode(out, errors="replace")
if loc_proc.returncode:
log("die", "SLOCCount failed", output.splitlines())
else:
log_sloc(output.splitlines())
# Pick up output left by red
try:
if os.path.exists("compile_commands.json"):
with open("compile_commands.json") as f:
red_output = json.load(f)
log("red", "red", output=red_output)
else:
log("die", "No red output found in dir '%s'" % os.getcwd())
except json.decoder.JSONDecodeError as e:
log("red", "red", output=[])
red_errors = []
for native in glob("/tmp/red-error-*"):
with open(native) as f:
lines = f.readlines()
red_errors.append({
"category": lines[0].strip(),
"pid": lines[1].strip(),
"info": "\n".join(lines[2:])
})
os.unlink(native)
log("red_errors", "red_errors", output=red_errors)
return proc.returncode
def main():
parser = get_argparser()
parser.add_argument("--abs-dir", dest="abs_dir", required=True)
parser.add_argument("--sysroot", default="sysroot")
args = parser.parse_args()
args.mirror_directory = "/mirror"
os.nice(10)
dump_build_information(args)
pkg_dir = os.path.basename(args.abs_dir)
if pkg_dir in os.listdir(args.sources_directory):
args.permanent_source_dir = os.path.join(args.sources_directory,
pkg_dir)
else:
die(
Status.failure, "No source directory in source volume: %s" %
args.sources_directory)
args.build_dir = os.path.join("/tmp", pkg_dir)
set_local_repository_location(args.toolchain_directory,
toolchain_repo_name())
if not os.path.isdir("/sysroot"):
os.makedirs("/sysroot")
copied_files = []
existing_files = []
for d in os.listdir("/toolchain_root"):
base = os.path.basename(d)
src = os.path.join("/toolchain_root", d)
dst = os.path.join("/sysroot", base)
# This can happen if we built the toolchain for the first time
# on this run. If we're using a pre-built toolchain, the file
# won't exist.
if os.path.lexists(dst):
existing_files.append(dst)
continue
if os.path.isfile(src):
copied_files.append((src, dst))
shutil.copyfile(src, dst)
elif os.path.isdir(src):
copied_files.append((src, dst))
shutil.copytree(src, dst)
copied_files = ["%s --> %s" % (src, dst) for (src, dst) in copied_files]
if copied_files:
log("info",
"Copied permanent toolchain into container-local sysroot",
output=copied_files)
if existing_files:
log("info",
"There were existing files in /sysroot, using those",
output=existing_files)
recursive_chown("/sysroot")
result = copy_and_build(args)
if result:
die(Status.failure)
paths_to_packages = create_hybrid_packages(args)
if not paths_to_packages:
die(Status.failure, "No hybrid packages were created.")
for path in paths_to_packages:
add_package_to_toolchain_repo(path, args.toolchain_directory)
die(Status.success)
