def linux_nightly():
return (
linux_build_task("Nightly build and upload").with_treeherder(
"Linux x64",
"Nightly").with_features("taskclusterProxy").with_scopes(
"secrets:get:project/servo/s3-upload-credentials")
# Not reusing the build made for WPT because it has debug assertions
.with_script(
"./mach build --release",
"./mach package --release",
"./mach upload-nightly linux --secret-from-taskcluster",
).with_artifacts("/repo/target/release/servo-tech-demo.tar.gz").
find_or_create("build.linux_x64_nightly" + CONFIG.task_id()))
def magicleap_nightly():
return (
magicleap_build_task("Nightly build and upload", "Release")
.with_features("taskclusterProxy")
.with_scopes("secrets:get:project/servo/s3-upload-credentials")
.with_script("""
./mach build --magicleap --release
env -u DYLD_LIBRARY_PATH ./mach package --magicleap --release
./mach upload-nightly magicleap --secret-from-taskcluster
""")
.with_artifacts("repo/target/magicleap/aarch64-linux-android/release/Servo.mpk")
.find_or_create("build.magicleap_nightly." + CONFIG.task_id())
)
def windows_arm64():
return (windows_build_task(
"UWP dev build", arch="arm64", package=False
).with_treeherder(
"Windows arm64", "UWP-Dev"
).with_features("taskclusterProxy").with_scopes(
"secrets:get:project/servo/windows-codesign-cert/latest"
).with_script(
"python mach build --dev --target=aarch64-uwp-windows-msvc",
"python mach package --dev --target aarch64-uwp-windows-msvc --uwp=arm64",
).with_artifacts(appx_artifact(
debug=True)).find_or_create("build.windows_uwp_arm64_dev." +
CONFIG.task_id()))
def windows_nightly():
return (
windows_build_task("Nightly build and upload")
.with_treeherder("Windows x64", "Nightly")
.with_features("taskclusterProxy")
.with_scopes("secrets:get:project/servo/s3-upload-credentials")
.with_script("mach fetch",
"mach build --release",
"mach package --release",
"mach upload-nightly windows-msvc --secret-from-taskcluster")
.with_artifacts("repo/target/release/msi/Servo.exe",
"repo/target/release/msi/Servo.zip")
.find_or_create("build.windows_x64_nightly." + CONFIG.task_id())
)
def macos_release_build_with_debug_assertions(priority=None):
return (macos_build_task(
"Release build, with debug assertions"
).with_treeherder(
"macOS x64", "Release+A"
).with_priority(priority).with_script("\n".join([
"./mach build --release --verbose --with-debug-assertions",
"./etc/ci/lockfile_changed.sh",
"tar -czf target.tar.gz" + " target/release/servo" +
" target/release/build/osmesa-src-*/output" +
" target/release/build/osmesa-src-*/out/src/gallium/targets/osmesa/.libs"
+ " target/release/build/osmesa-src-*/out/src/mapi/shared-glapi/.libs",
])).with_artifacts("repo/target.tar.gz").find_or_create(
"build.macos_x64_release_w_assertions." + CONFIG.task_id()))
def windows_unit():
return (windows_build_task("Dev build + unit tests").with_treeherder(
"Windows x64", "Unit"
).with_script(
# Not necessary as this would be done at the start of `build`,
# but this allows timing it separately.
"mach fetch",
"mach build --dev",
"mach test-unit",
"mach package --dev",
"mach build --dev --libsimpleservo",
).with_artifacts("repo/target/debug/msi/Servo.exe",
"repo/target/debug/msi/Servo.zip").find_or_create(
"build.windows_x64_dev." + CONFIG.task_id()))
def windows_arm64():
return (
windows_build_task("Dev build", arch="arm64", package=False)
.with_treeherder("Windows arm64")
.with_file_mount(
"https://dist.nuget.org/win-x86-commandline/latest/nuget.exe",
path="nuget.exe"
)
.with_script(
"%HOMEDRIVE%%HOMEPATH%\\nuget.exe install ANGLE.WindowsStore.Servo \
-Version 2.1.13 -o %HOMEDRIVE%%HOMEPATH%\\repo\\support\\hololens\\packages",
)
.with_script("python mach build --dev --uwp --win-arm64")
.find_or_create("build.windows_arm64_dev." + CONFIG.task_id())
)
def windows_x86():
return (
windows_build_task("Dev build", arch="x86")
.with_treeherder("Windows x86")
.with_env(**{
"VCVARSALL_PATH": "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\BuildTools\\VC\\Auxiliary\\Build"
})
.with_script(
"python mach build --dev --target i686-pc-windows-msvc",
"python mach package --dev --target i686-pc-windows-msvc",
)
.with_artifacts("repo/target/i686-pc-windows-msvc/debug/msi/Servo.exe",
"repo/target/i686-pc-windows-msvc/debug/msi/Servo.zip")
.find_or_create("build.windows_x86_dev." + CONFIG.task_id())
)
def uwp_nightly():
return (windows_build_task(
"Nightly UWP build and upload", package=False
).with_treeherder(
"Windows x64", "UWP Nightly"
).with_features("taskclusterProxy").with_scopes(
"secrets:get:project/servo/s3-upload-credentials"
).with_script(
"python mach build --release --target=x86_64-uwp-windows-msvc",
"python mach build --release --target=aarch64-uwp-windows-msvc",
"mach package --release --target=x86_64-uwp-windows-msvc --uwp=x64 --uwp=arm64",
"mach upload-nightly uwp --secret-from-taskcluster",
).with_artifacts(appx_artifact(debug=False)).with_max_run_time_minutes(
3 * 60).find_or_create("build.windows_uwp_nightlies." +
CONFIG.task_id()))
def macos_release_build():
return (
macos_build_task("Release build")
.with_treeherder("macOS x64", "Release")
.with_script("""
./mach build --release --verbose
./etc/ci/lockfile_changed.sh
tar -czf target.tar.gz \
target/release/servo \
target/release/build/osmesa-src-*/output \
target/release/build/osmesa-src-*/out/src/gallium/targets/osmesa/.libs \
target/release/build/osmesa-src-*/out/src/mapi/shared-glapi/.libs
""")
.with_artifacts("repo/target.tar.gz")
.find_or_create("build.macos_x64_release." + CONFIG.task_id())
)
def android_x86_wpt():
build_task = android_x86_release()
return (linux_task("WPT").with_treeherder(
"Android x86").with_provisioner_id("proj-servo").with_worker_type(
"docker-worker-kvm").with_capabilities(
privileged=True).with_scopes(
"project:servo:docker-worker-kvm:capability:privileged").
with_dockerfile(dockerfile_path("run-android-emulator")).with_repo(
).with_curl_artifact_script(
build_task, "servoapp.apk",
"target/android/i686-linux-android/release").with_script("""
./mach bootstrap-android --accept-all-licences --emulator-x86
./mach test-android-startup --release
./mach test-wpt-android --release \
/_mozilla/mozilla/DOMParser.html \
/_mozilla/mozilla/webgl/context_creation_error.html
""").find_or_create("android_x86_release." + CONFIG.task_id()))
def macos_nightly():
return (
macos_build_task("Nightly build and upload")
.with_treeherder("macOS x64", "Nightly")
.with_features("taskclusterProxy")
.with_scopes(
"secrets:get:project/servo/s3-upload-credentials",
"secrets:get:project/servo/github-homebrew-token",
)
.with_script(
"./mach build --release",
"./mach package --release",
"./mach upload-nightly mac --secret-from-taskcluster",
)
.with_artifacts("repo/target/release/servo-tech-demo.dmg")
.find_or_create("build.mac_x64_nightly." + CONFIG.task_id())
)
def linux_tidy_unit():
return (linux_build_task("Tidy + dev build + unit tests").with_treeherder(
"Linux x64", "Tidy+Unit").with_script("""
./mach test-tidy --no-progress --all
./mach build --dev
./mach test-unit
./mach package --dev
./mach build --dev --features canvas2d-raqote
./mach build --dev --features layout-2020
./mach build --dev --libsimpleservo
./mach test-tidy --no-progress --self-test
./etc/memory_reports_over_time.py --test
./etc/taskcluster/mock.py
./etc/ci/lockfile_changed.sh
./etc/ci/check_no_panic.sh
""").find_or_create("linux_unit." + CONFIG.task_id()))
def android_nightly():
return (android_build_task("Nightly build and upload").with_treeherder(
"Android Nightlies"
).with_features("taskclusterProxy").with_scopes(
"secrets:get:project/servo/s3-upload-credentials"
).with_script("""
./mach build --release --android
./mach package --release --android --maven
./mach build --release --target i686-linux-android
./mach package --release --target i686-linux-android --maven
./mach upload-nightly android --secret-from-taskcluster
./mach upload-nightly maven --secret-from-taskcluster
""").with_artifacts(
"/repo/target/android/armv7-linux-androideabi/release/servoapp.apk",
"/repo/target/android/armv7-linux-androideabi/release/servoview.aar",
"/repo/target/android/i686-linux-android/release/servoapp.apk",
"/repo/target/android/i686-linux-android/release/servoview.aar",
).find_or_create("build.android_nightlies." + CONFIG.task_id()))
def windows_unit(worker_type=None, cached=True):
task = (
windows_build_task("Dev build + unit tests", worker_type=worker_type).
with_treeherder("Windows x64", "Unit").with_script(
# Not necessary as this would be done at the start of `build`,
# but this allows timing it separately.
"mach fetch",
"mach build --dev",
"mach test-unit",
"mach smoketest --angle",
"mach package --dev",
"mach build --dev --libsimpleservo",
).with_artifacts("repo/target/debug/msi/Servo.exe",
"repo/target/debug/msi/Servo.zip"))
if cached:
return task.find_or_create("build.windows_x64_dev." + CONFIG.task_id())
else:
return task.create()
def linux_tidy_unit_docs():
return (
linux_build_task("Tidy + dev build + unit tests + docs")
.with_treeherder("Linux x64", "Tidy+Unit+Doc")
.with_script("""
./mach test-tidy --no-progress --all
./mach build --dev
./mach test-unit
./mach package --dev
./mach build --dev --features canvas2d-raqote
./mach build --dev --features layout-2020
./mach build --dev --libsimpleservo
./mach test-tidy --no-progress --self-test
./etc/memory_reports_over_time.py --test
./etc/taskcluster/mock.py
./etc/ci/lockfile_changed.sh
./etc/ci/check_no_panic.sh
rustup component add rust-docs
RUSTDOCFLAGS="--disable-minification" ./mach doc
(
cd target/doc
git init
git add .
git -c user.name="Taskcluster" -c user.email="" \
commit -q -m "Rebuild Servo documentation"
git bundle create docs.bundle HEAD
)
"""
# Because `rustdoc` needs metadata of dependency crates,
# `cargo doc` does almost all of the work that `cargo check` does.
# Therefore, when running them in this order the second command does very little
# and should finish quickly.
# The reverse order would not increase the total amount of work to do,
# but would reduce the amount of parallelism available.
"""
./mach check
""")
.with_artifacts("/repo/target/doc/docs.bundle")
.find_or_create("docs." + CONFIG.task_id())
)
def linux_wpt():
release_build_task = (
linux_build_task("Release build, with debug assertions")
.with_treeherder("Linux x64", "Release+A")
.with_script("""
./mach build --release --with-debug-assertions -p servo
./etc/ci/lockfile_changed.sh
tar -czf /target.tar.gz \
target/release/servo \
target/release/build/osmesa-src-*/output \
target/release/build/osmesa-src-*/out/lib/gallium
""")
.with_artifacts("/target.tar.gz")
.find_or_create("build.linux_x64_release~assertions" + CONFIG.task_id())
)
def linux_run_task(name):
return linux_task(name).with_dockerfile(dockerfile_path("run"))
wpt_chunks("Linux x64", linux_run_task, release_build_task, repo_dir="/repo",
total_chunks=2, processes=24)
def linux_tidy_unit():
return (linux_build_task("Tidy + dev build + unit tests").with_treeherder(
"Linux x64", "Tidy+Unit").with_max_run_time_minutes(75).with_script("""
./mach test-tidy --no-progress --all
python3 ./mach test-tidy --no-progress --all --no-wpt
python3 ./mach build --dev
python3 ./mach test-unit
python3 ./mach package --dev
python3 ./mach build --dev --features refcell_backtrace
python3 ./mach build --dev --features layout-2020
python3 ./mach build --dev --libsimpleservo
python3 ./mach build --dev -p servo-gst-plugin
python3 ./mach build --dev --media-stack=dummy
python3 ./mach test-tidy --no-progress --self-test
./etc/memory_reports_over_time.py --test
./etc/taskcluster/mock.py
./etc/ci/lockfile_changed.sh
./etc/ci/check_no_panic.sh
""").find_or_create("linux_unit." + CONFIG.task_id()))
def upload_docs():
docs_build_task_id = decisionlib.Task.find("docs." + CONFIG.task_id())
return (linux_task("Upload docs to GitHub Pages").with_treeherder(
"Linux x64", "DocUpload").with_dockerfile(
dockerfile_path("base")).with_curl_artifact_script(
docs_build_task_id,
"docs.bundle").with_features("taskclusterProxy").with_scopes(
"secrets:get:project/servo/doc.servo.org").with_env(
PY="""if 1:
import urllib, json
url = "http://taskcluster/secrets/v1/secret/project/servo/doc.servo.org"
token = json.load(urllib.urlopen(url))["secret"]["token"]
open("/root/.git-credentials", "w").write("https://*****:*****@github.com/" % token)
""").with_script("""
python -c "$PY"
git init --bare
git config credential.helper store
git fetch --quiet docs.bundle
git push --force https://github.com/servo/doc.servo.org FETCH_HEAD:gh-pages
""").create())
def windows_unit(cached=True):
task = (
windows_build_task("Dev build + unit tests").
with_treeherder("Windows x64", "Unit").with_script(
# Not necessary as this would be done at the start of `build`,
# but this allows timing it separately.
"mach fetch",
"mach build --dev",
"mach test-unit",
"mach smoketest --angle",
"mach package --dev",
"mach build --dev --libsimpleservo",
# We're getting link errors on windows, due to the x11 feature being
# enabled on gstreamer-gl. https://github.com/servo/media/pull/304/
"mach build --dev --media-stack=dummy -p servo-gst-plugin",
).with_artifacts("repo/target/debug/msi/Servo.exe",
"repo/target/debug/msi/Servo.zip"))
if cached:
return task.find_or_create("build.windows_x64_dev." + CONFIG.task_id())
else:
return task.create()
def windows_unit(cached=True):
task = (
windows_build_task("Dev build + unit tests").
with_treeherder("Windows x64", "Unit").with_script(
# Not necessary as this would be done at the start of `build`,
# but this allows timing it separately.
"mach fetch",
"mach build --dev",
"mach test-unit",
"mach smoketest --angle",
"mach package --dev",
"mach build --dev --libsimpleservo",
# The GStreamer plugin currently doesn't support Windows
# https://github.com/servo/servo/issues/25353
# "mach build --dev -p servo-gst-plugin",
).with_artifacts("repo/target/debug/msi/Servo.exe",
"repo/target/debug/msi/Servo.zip"))
if cached:
return task.find_or_create("build.windows_x64_dev." + CONFIG.task_id())
else:
return task.create()
def update_wpt():
build_task = macos_release_build()
update_task = (macos_task("WPT update").with_python2().with_treeherder(
"macOS x64",
"WPT update").with_features("taskclusterProxy").with_scopes(
"secrets:get:project/servo/wpt-sync"
).with_index_and_artifacts_expire_in(
log_artifacts_expire_in).with_max_run_time_minutes(5 * 60))
return (
with_homebrew(update_task, [
"etc/taskcluster/macos/Brewfile-wpt",
"etc/taskcluster/macos/Brewfile-gstreamer",
])
# Pushing the new changes to the git remote requires a full repo clone.
.with_repo(shallow=False).with_curl_artifact_script(
build_task, "target.tar.gz").with_script("""
export PKG_CONFIG_PATH="$(brew --prefix libffi)/lib/pkgconfig/"
tar -xzf target.tar.gz
./etc/ci/update-wpt-checkout fetch-and-update-expectations
./etc/ci/update-wpt-checkout open-pr
./etc/ci/update-wpt-checkout cleanup
""").find_or_create("wpt_update." + CONFIG.task_id()))
def magicleap_dev():
return (magicleap_build_task("Dev build", "Dev").with_script("""
./mach build --magicleap --dev
env -u DYLD_LIBRARY_PATH ./mach package --magicleap --dev
""").find_or_create("build.magicleap_dev." + CONFIG.task_id()))
def wpt_chunks(platform,
make_chunk_task,
build_task,
total_chunks,
processes,
repo_dir,
chunks="all"):
if chunks == "all":
chunks = [n + 1 for n in range(total_chunks)]
for this_chunk in chunks:
task = (make_chunk_task(
"WPT chunk %s / %s" % (this_chunk, total_chunks)).with_treeherder(
platform,
"WPT-%s" % this_chunk).with_repo().with_curl_artifact_script(
build_task,
"target.tar.gz").with_script("tar -xzf target.tar.gz").
with_index_and_artifacts_expire_in(log_artifacts_expire_in).
with_max_run_time_minutes(90).with_env(
TOTAL_CHUNKS=str(total_chunks),
THIS_CHUNK=str(this_chunk),
PROCESSES=str(processes),
GST_DEBUG="3",
))
if this_chunk == chunks[-1]:
task.name += " + extra"
task.extra["treeherder"]["symbol"] += "+"
task.with_script("""
./mach test-wpt-failure
time ./mach test-wpt --release --binary-arg=--multiprocess \
--processes $PROCESSES \
--log-raw test-wpt-mp.log \
--log-errorsummary wpt-mp-errorsummary.log \
eventsource \
| cat
time ./mach test-wpt --release --product=servodriver --headless \
tests/wpt/mozilla/tests/mozilla/DOMParser.html \
tests/wpt/mozilla/tests/css/per_glyph_font_fallback_a.html \
tests/wpt/mozilla/tests/css/img_simple.html \
tests/wpt/mozilla/tests/mozilla/secure.https.html \
| cat
time ./mach test-wpt --release --processes $PROCESSES --product=servodriver \
--headless --log-raw test-bluetooth.log \
--log-errorsummary bluetooth-errorsummary.log \
bluetooth \
| cat
time ./mach test-wpt --release --processes $PROCESSES --timeout-multiplier=4 \
--headless --log-raw test-wdspec.log \
--log-errorsummary wdspec-errorsummary.log \
--always-succeed \
webdriver \
| cat
./mach filter-intermittents \
wdspec-errorsummary.log \
--log-intermittents intermittents.log \
--log-filteredsummary filtered-wdspec-errorsummary.log \
--tracker-api default \
--reporter-api default
""")
# `test-wpt` is piped into `cat` so that stdout is not a TTY
# and wptrunner does not use "interactive mode" formatting:
# https://github.com/servo/servo/issues/22438
task.with_script("""
./mach test-wpt \
--release \
--processes $PROCESSES \
--total-chunks "$TOTAL_CHUNKS" \
--this-chunk "$THIS_CHUNK" \
--log-raw test-wpt.log \
--log-errorsummary wpt-errorsummary.log \
--always-succeed \
| cat
./mach filter-intermittents \
wpt-errorsummary.log \
--log-intermittents intermittents.log \
--log-filteredsummary filtered-wpt-errorsummary.log \
--tracker-api default \
--reporter-api default
""")
task.with_artifacts(*[
"%s/%s" % (repo_dir, word) for script in task.scripts
for word in script.split() if word.endswith(".log")
])
platform_id = platform.replace(" ", "_").lower()
task.find_or_create("%s_wpt_%s.%s" %
(platform_id, this_chunk, CONFIG.task_id()))
def linux_release():
return (linux_build_task("Release build").with_treeherder(
"Linux x64", "Release").with_script(
"./mach build --release",
"./mach package --release",
).find_or_create("build.linux_x64_release" + CONFIG.task_id()))
def windows_arm64():
return (windows_build_task(
"UWP dev build", arch="arm64",
package=False).with_treeherder("Windows arm64").with_script(
"python mach build --dev --uwp --win-arm64").find_or_create(
"build.windows_uwp_arm64_dev." + CONFIG.task_id()))
def windows_uwp_x64():
return (windows_build_task("UWP dev build", package=False).with_treeherder(
"Windows x64").with_script("mach build --dev --uwp").find_or_create(
"build.windows_uwp_x64_dev." + CONFIG.task_id()))
def wpt_chunks(platform,
make_chunk_task,
build_task,
total_chunks,
processes,
repo_dir,
chunks="all",
layout_2020=False):
if layout_2020:
start = 1 # Skip the "extra" WPT testing, a.k.a. chunk 0
name_prefix = "Layout 2020 "
job_id_prefix = "2020-"
args = "--layout-2020"
else:
start = 0
name_prefix = ""
job_id_prefix = ""
args = ""
if chunks == "all":
chunks = range(start, total_chunks + 1)
for this_chunk in chunks:
task = (make_chunk_task("{}WPT chunk {:0{width}} / {}".format(
name_prefix,
this_chunk,
total_chunks,
width=len(str(total_chunks)),
)).with_treeherder(
platform,
"WPT-%s" % this_chunk,
group_symbol=job_id_prefix + "WPT",
group_name=name_prefix +
"web-platform-tests").with_curl_artifact_script(
build_task,
"target.tar.gz").with_script("tar -xzf target.tar.gz").
with_index_and_artifacts_expire_in(log_artifacts_expire_in).
with_max_run_time_minutes(90).with_env(
TOTAL_CHUNKS=str(total_chunks),
THIS_CHUNK=str(this_chunk),
PROCESSES=str(processes),
WPT_ARGS=args,
GST_DEBUG="3",
))
# `test-wpt` is piped into `cat` so that stdout is not a TTY
# and wptrunner does not use "interactive mode" formatting:
# https://github.com/servo/servo/issues/22438
if this_chunk == 0:
task.with_script("""
./mach test-wpt-failure
time python2 ./mach test-wpt --release --binary-arg=--multiprocess \
--processes $PROCESSES \
--log-raw test-wpt-mp.log \
--log-errorsummary wpt-mp-errorsummary.log \
eventsource \
| cat
time env PYTHONIOENCODING=utf-8 python3 ./mach test-wpt --release --binary-arg=--multiprocess \
--processes $PROCESSES \
--log-raw test-wpt-mp.log \
--log-errorsummary wpt-mp-errorsummary.log \
eventsource \
| cat
time ./mach test-wpt --release --product=servodriver --headless \
tests/wpt/mozilla/tests/mozilla/DOMParser.html \
tests/wpt/mozilla/tests/css/per_glyph_font_fallback_a.html \
tests/wpt/mozilla/tests/css/img_simple.html \
tests/wpt/mozilla/tests/mozilla/secure.https.html \
| cat
time ./mach test-wpt --release --processes $PROCESSES --product=servodriver \
--headless --log-raw test-bluetooth.log \
--log-errorsummary bluetooth-errorsummary.log \
bluetooth \
| cat
time ./mach test-wpt --release --processes $PROCESSES --timeout-multiplier=4 \
--headless --log-raw test-wdspec.log \
--log-servojson wdspec-jsonsummary.log \
--always-succeed \
webdriver \
| cat
./mach filter-intermittents \
wdspec-jsonsummary.log \
--log-intermittents intermittents.log \
--log-filteredsummary filtered-wdspec-errorsummary.log \
--tracker-api default \
--reporter-api default
""")
else:
task.with_script("""
./mach test-wpt \
--release \
$WPT_ARGS \
--processes $PROCESSES \
--total-chunks "$TOTAL_CHUNKS" \
--this-chunk "$THIS_CHUNK" \
--log-raw test-wpt.log \
--log-servojson wpt-jsonsummary.log \
--always-succeed \
| cat
./mach filter-intermittents \
wpt-jsonsummary.log \
--log-intermittents intermittents.log \
--log-filteredsummary filtered-wpt-errorsummary.log \
--tracker-api default \
--reporter-api default
""")
task.with_artifacts(*[
"%s/%s" % (repo_dir, word) for script in task.scripts
for word in script.split() if word.endswith(".log")
])
task.find_or_create("%s_%swpt_%s.%s" % (
platform.replace(" ", "_").lower(),
job_id_prefix.replace("-", "_"),
this_chunk,
CONFIG.task_id(),
))
