def linux_docs_check():
return (
linux_build_task("Docs + check")
.with_treeherder("Linux x64", "Doc+Check")
.with_script("""
RUSTDOCFLAGS="--disable-minification" ./mach doc
(
cd target/doc
git init
git add .
git -c user.name="Taskcluster" -c user.email="" \
commit -q -m "Rebuild Servo documentation"
git bundle create docs.bundle HEAD
)
"""
# Because `rustdoc` needs metadata of dependency crates,
# `cargo doc` does almost all of the work that `cargo check` does.
# Therefore, when running them in this order the second command does very little
# and should finish quickly.
# The reverse order would not increase the total amount of work to do,
# but would reduce the amount of parallelism available.
"""
./mach check
""")
.with_artifacts("/repo/target/doc/docs.bundle")
.find_or_create("docs." + CONFIG.tree_hash())
)
def linux_release_build_with_debug_assertions(layout_2020):
if layout_2020:
name_prefix = "Layout 2020 "
build_args = "--with-layout-2020"
index_key_suffix = "_2020"
treeherder_prefix = "2020-"
else:
name_prefix = ""
build_args = ""
index_key_suffix = ""
treeherder_prefix = ""
return (linux_build_task(
name_prefix + "Release build, with debug assertions").with_treeherder(
"Linux x64", treeherder_prefix + "Release+A").with_script("""
time ./mach rustc -V
time ./mach fetch
./mach build --release --with-debug-assertions %s -p servo
./etc/ci/lockfile_changed.sh
tar -czf /target.tar.gz \
target/release/servo \
resources
sccache --show-stats
""" % build_args).with_artifacts("/target.tar.gz").find_or_create(
"build.linux_x64%s_release_w_assertions.%s" % (
index_key_suffix,
CONFIG.tree_hash(),
)))
def windows_unit(cached=True, rdp=False):
task = (
windows_build_task("Dev build + unit tests", rdp=rdp)
.with_treeherder("Windows x64", "Unit")
.with_script(
# Not necessary as this would be done at the start of `build`,
# but this allows timing it separately.
"mach fetch",
"mach build --dev",
"mach test-unit",
"mach smoketest --angle",
"mach package --dev",
"mach build --dev --libsimpleservo",
# The GStreamer plugin currently doesn't support Windows
# https://github.com/servo/servo/issues/25353
# "mach build --dev -p servo-gst-plugin",
)
.with_artifacts("repo/target/debug/msi/Servo.exe",
"repo/target/debug/msi/Servo.zip")
)
if cached:
return task.find_or_create("build.windows_x64_dev." + CONFIG.tree_hash())
else:
return task.create()
def upload_docs():
docs_build_task_id = decisionlib.Task.find("docs." + CONFIG.tree_hash())
return (
linux_task("Upload docs to GitHub Pages")
.with_treeherder("Linux x64", "DocUpload")
.with_dockerfile(dockerfile_path("base"))
.with_curl_artifact_script(docs_build_task_id, "docs.bundle")
.with_features("taskclusterProxy")
.with_scopes("secrets:get:project/servo/doc.servo.org")
.with_env(PY="""if 1:
import urllib, json, os
root_url = os.environ["TASKCLUSTER_PROXY_URL"]
url = root_url + "/api/secrets/v1/secret/project/servo/doc.servo.org"
token = json.load(urllib.urlopen(url))["secret"]["token"]
open("/root/.git-credentials", "w").write("https://*****:*****@github.com/" % token)
""")
.with_script("""
python -c "$PY"
git init --bare
git config credential.helper store
git fetch --quiet docs.bundle
git push --force https://github.com/servo/doc.servo.org FETCH_HEAD:gh-pages
""")
.create()
)
def linux_tidy_unit():
return (
linux_build_task("Tidy + dev build + unit tests")
.with_treeherder("Linux x64", "Tidy+Unit")
.with_max_run_time_minutes(75)
.with_script("""
./mach test-tidy --no-progress --all
python3 ./mach test-tidy --no-progress --all --no-wpt
python3 ./mach build --dev
python3 ./mach test-unit
python3 ./mach package --dev
python3 ./mach build --dev --features refcell_backtrace
python3 ./mach build --dev --features layout-2020
python3 ./mach build --dev --libsimpleservo
python3 ./mach build --dev -p servo-gst-plugin
python3 ./mach build --dev --media-stack=dummy
python3 ./mach test-tidy --no-progress --self-test
./etc/memory_reports_over_time.py --test
./etc/taskcluster/mock.py
./etc/ci/lockfile_changed.sh
./etc/ci/check_no_panic.sh
""")
.find_or_create("linux_unit." + CONFIG.tree_hash())
)
def macos_unit():
return (macos_build_task("Dev build + unit tests").with_treeherder(
"macOS x64", "Unit").with_script("""
./mach build --dev --verbose
./mach test-unit
./mach package --dev
./etc/ci/lockfile_changed.sh
""").find_or_create("macos_unit." + CONFIG.tree_hash()))
def layout_2020_regressions_report():
return (linux_task("Layout 2020 regressions report").with_treeherder(
"Linux x64", "RegressionsReport").with_dockerfile(
dockerfile_path("base")).with_repo_bundle().with_script(
"python3 tests/wpt/reftests-report/gen.py %s %s" %
(CONFIG.tree_hash(), CONFIG.git_sha)).
with_index_and_artifacts_expire_in(log_artifacts_expire_in).
with_artifacts("/repo/tests/wpt/reftests-report/report.html").
with_index_at("layout-2020-regressions-report").create())
def macos_unit():
return (macos_build_task("Dev build + unit tests").with_treeherder(
"macOS x64", "Unit").with_script("""
python3 ./mach build --dev --verbose
python3 ./mach test-unit
python3 ./mach package --dev
./etc/ci/macos_package_smoketest.sh target/debug/servo-tech-demo.dmg
./etc/ci/lockfile_changed.sh
""").find_or_create("macos_unit." + CONFIG.tree_hash()))
def linux_release():
return (
linux_build_task("Release build")
.with_treeherder("Linux x64", "Release")
.with_script(
"./mach build --release",
"./mach package --release",
)
.find_or_create("build.linux_x64_release" + CONFIG.tree_hash())
)
def macos_release_build_with_debug_assertions(priority=None):
return (macos_build_task(
"Release build, with debug assertions").with_treeherder(
"macOS x64", "Release+A"
).with_priority(priority).with_script("\n".join([
"python3 ./mach build --release --verbose --with-debug-assertions",
"./etc/ci/lockfile_changed.sh",
"tar -czf target.tar.gz" + " target/release/servo" +
" target/release/*.dylib" + " resources",
])).with_artifacts("repo/target.tar.gz").find_or_create(
"build.macos_x64_release_w_assertions." + CONFIG.tree_hash()))
def macos_nightly():
return (macos_build_task("Nightly build and upload").with_treeherder(
"macOS x64", "Nightly").with_features("taskclusterProxy").with_scopes(
"secrets:get:project/servo/s3-upload-credentials",
"secrets:get:project/servo/github-homebrew-token",
).with_script(
"./mach build --release",
"./mach package --release",
"./mach upload-nightly mac --secret-from-taskcluster",
).with_artifacts("repo/target/release/servo-tech-demo.dmg").
find_or_create("build.mac_x64_nightly." + CONFIG.tree_hash()))
def windows_nightly(rdp=False):
return (windows_build_task(
"Nightly build and upload", rdp=rdp).with_treeherder(
"Windows x64", "Nightly"
).with_features("taskclusterProxy").with_scopes(
"secrets:get:project/servo/s3-upload-credentials").with_script(
"mach fetch", "mach build --release", "mach package --release",
"mach upload-nightly windows-msvc --secret-from-taskcluster").
with_artifacts("repo/target/release/msi/Servo.exe",
"repo/target/release/msi/Servo.zip").find_or_create(
"build.windows_x64_nightly." +
CONFIG.tree_hash()))
def windows_arm64(rdp=False):
return (windows_build_task(
"UWP dev build", arch="arm64", package=False, rdp=rdp
).with_treeherder(
"Windows arm64", "UWP-Dev"
).with_features("taskclusterProxy").with_scopes(
"secrets:get:project/servo/windows-codesign-cert/latest"
).with_script(
"python mach build --dev --target=aarch64-uwp-windows-msvc",
"python mach package --dev --target aarch64-uwp-windows-msvc --uwp=arm64",
).with_artifacts(appx_artifact).find_or_create(
"build.windows_uwp_arm64_dev." + CONFIG.tree_hash()))
def linux_nightly():
return (
linux_build_task("Nightly build and upload").with_treeherder(
"Linux x64",
"Nightly").with_features("taskclusterProxy").with_scopes(
"secrets:get:project/servo/s3-upload-credentials")
# Not reusing the build made for WPT because it has debug assertions
.with_script(
"./mach build --release",
"./mach package --release",
"./mach upload-nightly linux --secret-from-taskcluster",
).with_artifacts("/repo/target/release/servo-tech-demo.tar.gz").
find_or_create("build.linux_x64_nightly" + CONFIG.tree_hash()))
def linux_docs_check():
return (
linux_build_task("Check").with_treeherder(
"Linux x64", "Check").with_script(
'RUSTDOCFLAGS="--disable-minification" python3 ./mach doc')
# Because `rustdoc` needs metadata of dependency crates,
# `cargo doc` does almost all of the work that `cargo check` does.
# Therefore, when running them in this order the second command does very little
# and should finish quickly.
# The reverse order would not increase the total amount of work to do,
# but would reduce the amount of parallelism available.
.with_script("python3 ./mach check").find_or_create("check." +
CONFIG.tree_hash())
)
def macos_release_build_with_debug_assertions(priority=None):
return (macos_build_task(
"Release build, with debug assertions"
).with_treeherder(
"macOS x64", "Release+A"
).with_priority(priority).with_script("\n".join([
"./mach build --release --verbose --with-debug-assertions",
"./etc/ci/lockfile_changed.sh",
"tar -czf target.tar.gz" + " target/release/servo" +
" target/release/build/osmesa-src-*/output" +
" target/release/build/osmesa-src-*/out/src/gallium/targets/osmesa/.libs"
+ " target/release/build/osmesa-src-*/out/src/mapi/shared-glapi/.libs",
])).with_artifacts("repo/target.tar.gz").find_or_create(
"build.macos_x64_release_w_assertions." + CONFIG.tree_hash()))
def windows_uwp_x64():
return (
windows_build_task("UWP dev build", package=False)
.with_treeherder("Windows x64", "UWP-Dev")
.with_features("taskclusterProxy")
.with_scopes("secrets:get:project/servo/windows-codesign-cert/latest")
.with_script(
"python mach build --dev --target=x86_64-uwp-windows-msvc",
"python mach package --dev --target=x86_64-uwp-windows-msvc --uwp=x64",
"python mach test-tidy --force-cpp --no-wpt",
)
.with_artifacts(appx_artifact(debug=True))
.find_or_create("build.windows_uwp_x64_dev." + CONFIG.tree_hash())
)
def uwp_nightly(rdp=False):
return (windows_build_task(
"Nightly UWP build and upload", package=False, rdp=rdp
).with_treeherder(
"Windows x64", "UWP-Nightly"
).with_features("taskclusterProxy").with_scopes(
"secrets:get:project/servo/s3-upload-credentials",
"secrets:get:project/servo/windows-codesign-cert/latest",
).with_script(
"python mach build --release --target=x86_64-uwp-windows-msvc",
"python mach build --release --target=aarch64-uwp-windows-msvc",
"mach package --release --target=x86_64-uwp-windows-msvc --uwp=x64 --uwp=arm64",
"mach upload-nightly uwp --secret-from-taskcluster",
).with_artifacts(appx_artifact).with_max_run_time_minutes(
3 * 60).find_or_create("build.windows_uwp_nightlies." +
CONFIG.tree_hash()))
def wpt_chunks(platform, make_chunk_task, build_task, total_chunks, processes,
repo_dir, chunks="all", layout_2020=False):
if layout_2020:
start = 1 # Skip the "extra" WPT testing, a.k.a. chunk 0
name_prefix = "Layout 2020 "
job_id_prefix = "2020-"
args = "--layout-2020"
else:
start = 0
name_prefix = ""
job_id_prefix = ""
args = ""
if chunks == "all":
chunks = range(start, total_chunks + 1)
for this_chunk in chunks:
task = (
make_chunk_task("{}WPT chunk {:0{width}} / {}".format(
name_prefix,
this_chunk,
total_chunks,
width=len(str(total_chunks)),
))
.with_treeherder(
platform,
"WPT-%s" % this_chunk,
group_symbol=job_id_prefix + "WPT",
group_name=name_prefix + "web-platform-tests"
)
.with_curl_artifact_script(build_task, "target.tar.gz")
.with_script("tar -xzf target.tar.gz")
.with_index_and_artifacts_expire_in(log_artifacts_expire_in)
.with_max_run_time_minutes(90)
.with_env(
TOTAL_CHUNKS=str(total_chunks),
THIS_CHUNK=str(this_chunk),
PROCESSES=str(processes),
WPT_ARGS=args,
GST_DEBUG="3",
)
)
# `test-wpt` is piped into `cat` so that stdout is not a TTY
# and wptrunner does not use "interactive mode" formatting:
# https://github.com/servo/servo/issues/22438
if this_chunk == 0:
task.with_script("""
./mach test-wpt-failure
time python2 ./mach test-wpt --release --binary-arg=--multiprocess \
--processes $PROCESSES \
--log-raw test-wpt-mp.log \
--log-errorsummary wpt-mp-errorsummary.log \
eventsource \
| cat
time env PYTHONIOENCODING=utf-8 python3 ./mach test-wpt --release --binary-arg=--multiprocess \
--processes $PROCESSES \
--log-raw test-wpt-mp.log \
--log-errorsummary wpt-mp-errorsummary.log \
eventsource \
| cat
time ./mach test-wpt --release --product=servodriver --headless \
tests/wpt/mozilla/tests/mozilla/DOMParser.html \
tests/wpt/mozilla/tests/css/per_glyph_font_fallback_a.html \
tests/wpt/mozilla/tests/css/img_simple.html \
tests/wpt/mozilla/tests/mozilla/secure.https.html \
| cat
time ./mach test-wpt --release --processes $PROCESSES --product=servodriver \
--headless --log-raw test-bluetooth.log \
--log-errorsummary bluetooth-errorsummary.log \
bluetooth \
| cat
time ./mach test-wpt --release --processes $PROCESSES --timeout-multiplier=4 \
--headless --log-raw test-wdspec.log \
--log-servojson wdspec-jsonsummary.log \
--always-succeed \
webdriver \
| cat
./mach filter-intermittents \
wdspec-jsonsummary.log \
--log-intermittents intermittents.log \
--log-filteredsummary filtered-wdspec-errorsummary.log \
--tracker-api default \
--reporter-api default
""")
else:
task.with_script("""
./mach test-wpt \
--release \
$WPT_ARGS \
--processes $PROCESSES \
--total-chunks "$TOTAL_CHUNKS" \
--this-chunk "$THIS_CHUNK" \
--log-raw test-wpt.log \
--log-servojson wpt-jsonsummary.log \
--always-succeed \
| cat
./mach filter-intermittents \
wpt-jsonsummary.log \
--log-intermittents intermittents.log \
--log-filteredsummary filtered-wpt-errorsummary.log \
--tracker-api default \
--reporter-api default
""")
task.with_artifacts(*[
"%s/%s" % (repo_dir, word)
for script in task.scripts
for word in script.split()
if word.endswith(".log")
])
task.find_or_create("%s_%swpt_%s.%s" % (
platform.replace(" ", "_").lower(),
job_id_prefix.replace("-", "_"),
this_chunk,
CONFIG.tree_hash(),
))
def wpt_chunks(platform,
make_chunk_task,
build_task,
total_chunks,
processes,
repo_dir,
chunks="all",
layout_2020=False,
run_webgpu=False):
if layout_2020:
start = 1 # Skip the "extra" WPT testing, a.k.a. chunk 0
name_prefix = "Layout 2020 "
job_id_prefix = "2020-"
args = ["--layout-2020"]
else:
start = 0
name_prefix = ""
job_id_prefix = ""
args = []
# Our Mac CI runs on machines with an Intel 4000 GPU, so need to work around
# https://github.com/servo/webrender/wiki/Driver-issues#bug-1570736---texture-swizzling-affects-wrap-modes-on-some-intel-gpus
if platform == "macOS x64":
args += ["--pref gfx.texture-swizzling.enabled=false"]
if chunks == "all":
chunks = range(start, total_chunks + 1)
for this_chunk in chunks:
task = (make_chunk_task("{}WPT chunk {:0{width}} / {}".format(
name_prefix,
this_chunk,
total_chunks,
width=len(str(total_chunks)),
)).with_treeherder(
platform,
"WPT-%s" % this_chunk,
group_symbol=job_id_prefix + "WPT",
group_name=name_prefix +
"web-platform-tests").with_curl_artifact_script(
build_task,
"target.tar.gz").with_script("tar -xzf target.tar.gz").
with_index_and_artifacts_expire_in(log_artifacts_expire_in).
with_max_run_time_minutes(90).with_env(
TOTAL_CHUNKS=str(total_chunks),
THIS_CHUNK=str(this_chunk),
PROCESSES=str(processes),
WPT_ARGS=" ".join(args),
GST_DEBUG="3",
))
# `test-wpt` is piped into `cat` so that stdout is not a TTY
# and wptrunner does not use "interactive mode" formatting:
# https://github.com/servo/servo/issues/22438
if this_chunk == 0:
if run_webgpu:
webgpu_script = """
time python3 ./mach test-wpt _webgpu --release --processes $PROCESSES \
--headless --log-raw test-webgpu.log --always-succeed \
--log-errorsummary webgpu-errorsummary.log \
| cat
python3 ./mach filter-intermittents \
webgpu-errorsummary.log \
--log-intermittents webgpu-intermittents.log \
--log-filteredsummary filtered-webgpu-errorsummary.log \
--tracker-api default \
--reporter-api default
""" # pragma: no cover
else:
webgpu_script = ""
task.with_script("""
time python3 ./mach test-wpt --release --binary-arg=--multiprocess \
--processes $PROCESSES \
--log-raw test-wpt-mp.log \
--log-errorsummary wpt-mp-errorsummary.log \
eventsource \
| cat
time env PYTHONIOENCODING=utf-8 python3 ./mach test-wpt --release \
--processes $PROCESSES \
--log-raw test-wpt-py3.log \
--log-errorsummary wpt-py3-errorsummary.log \
--always-succeed \
url \
| cat
python3 ./mach filter-intermittents \
wpt-py3-errorsummary.log \
--log-intermittents wpt-py3-intermittents.log \
--log-filteredsummary filtered-py3-errorsummary.log \
--tracker-api default \
--reporter-api default
time python3 ./mach test-wpt --release --product=servodriver --headless \
tests/wpt/mozilla/tests/mozilla/DOMParser.html \
tests/wpt/mozilla/tests/css/per_glyph_font_fallback_a.html \
tests/wpt/mozilla/tests/css/img_simple.html \
tests/wpt/mozilla/tests/mozilla/secure.https.html \
| cat
time python3 ./mach test-wpt --release --processes $PROCESSES --product=servodriver \
--headless --log-raw test-bluetooth.log \
--log-errorsummary bluetooth-errorsummary.log \
bluetooth \
| cat
time python3 ./mach test-wpt --release --processes $PROCESSES --timeout-multiplier=4 \
--headless --log-raw test-wdspec.log \
--log-servojson wdspec-jsonsummary.log \
--always-succeed \
webdriver \
| cat
python3 ./mach filter-intermittents \
wdspec-jsonsummary.log \
--log-intermittents intermittents.log \
--log-filteredsummary filtered-wdspec-errorsummary.log \
--tracker-api default \
--reporter-api default
""" + webgpu_script)
else:
task.with_script("""
python3 ./mach test-wpt \
--release \
$WPT_ARGS \
--processes $PROCESSES \
--total-chunks "$TOTAL_CHUNKS" \
--this-chunk "$THIS_CHUNK" \
--log-raw test-wpt.log \
--log-servojson wpt-jsonsummary.log \
--always-succeed \
| cat
python3 ./mach filter-intermittents \
wpt-jsonsummary.log \
--log-intermittents intermittents.log \
--log-filteredsummary filtered-wpt-errorsummary.log \
--tracker-api default \
--reporter-api default
""")
all_artifacts = set([
"%s/%s" % (repo_dir, word) for script in task.scripts
for word in script.split() if word.endswith(".log")
])
task.with_artifacts(*all_artifacts)
task.find_or_create("%s_%swpt_%s.%s" % (
platform.replace(" ", "_").lower(),
job_id_prefix.replace("-", "_"),
this_chunk,
CONFIG.tree_hash(),
))
