def linux_docs_check(): return ( linux_build_task("Docs + check") .with_treeherder("Linux x64", "Doc+Check") .with_script(""" RUSTDOCFLAGS="--disable-minification" ./mach doc ( cd target/doc git init git add . git -c user.name="Taskcluster" -c user.email="" \ commit -q -m "Rebuild Servo documentation" git bundle create docs.bundle HEAD ) """ # Because `rustdoc` needs metadata of dependency crates, # `cargo doc` does almost all of the work that `cargo check` does. # Therefore, when running them in this order the second command does very little # and should finish quickly. # The reverse order would not increase the total amount of work to do, # but would reduce the amount of parallelism available. """ ./mach check """) .with_artifacts("/repo/target/doc/docs.bundle") .find_or_create("docs." + CONFIG.tree_hash()) )
def linux_release_build_with_debug_assertions(layout_2020): if layout_2020: name_prefix = "Layout 2020 " build_args = "--with-layout-2020" index_key_suffix = "_2020" treeherder_prefix = "2020-" else: name_prefix = "" build_args = "" index_key_suffix = "" treeherder_prefix = "" return (linux_build_task( name_prefix + "Release build, with debug assertions").with_treeherder( "Linux x64", treeherder_prefix + "Release+A").with_script(""" time ./mach rustc -V time ./mach fetch ./mach build --release --with-debug-assertions %s -p servo ./etc/ci/lockfile_changed.sh tar -czf /target.tar.gz \ target/release/servo \ resources sccache --show-stats """ % build_args).with_artifacts("/target.tar.gz").find_or_create( "build.linux_x64%s_release_w_assertions.%s" % ( index_key_suffix, CONFIG.tree_hash(), )))
def windows_unit(cached=True, rdp=False): task = ( windows_build_task("Dev build + unit tests", rdp=rdp) .with_treeherder("Windows x64", "Unit") .with_script( # Not necessary as this would be done at the start of `build`, # but this allows timing it separately. "mach fetch", "mach build --dev", "mach test-unit", "mach smoketest --angle", "mach package --dev", "mach build --dev --libsimpleservo", # The GStreamer plugin currently doesn't support Windows # https://github.com/servo/servo/issues/25353 # "mach build --dev -p servo-gst-plugin", ) .with_artifacts("repo/target/debug/msi/Servo.exe", "repo/target/debug/msi/Servo.zip") ) if cached: return task.find_or_create("build.windows_x64_dev." + CONFIG.tree_hash()) else: return task.create()
def upload_docs(): docs_build_task_id = decisionlib.Task.find("docs." + CONFIG.tree_hash()) return ( linux_task("Upload docs to GitHub Pages") .with_treeherder("Linux x64", "DocUpload") .with_dockerfile(dockerfile_path("base")) .with_curl_artifact_script(docs_build_task_id, "docs.bundle") .with_features("taskclusterProxy") .with_scopes("secrets:get:project/servo/doc.servo.org") .with_env(PY="""if 1: import urllib, json, os root_url = os.environ["TASKCLUSTER_PROXY_URL"] url = root_url + "/api/secrets/v1/secret/project/servo/doc.servo.org" token = json.load(urllib.urlopen(url))["secret"]["token"] open("/root/.git-credentials", "w").write("https://*****:*****@github.com/" % token) """) .with_script(""" python -c "$PY" git init --bare git config credential.helper store git fetch --quiet docs.bundle git push --force https://github.com/servo/doc.servo.org FETCH_HEAD:gh-pages """) .create() )
def linux_tidy_unit(): return ( linux_build_task("Tidy + dev build + unit tests") .with_treeherder("Linux x64", "Tidy+Unit") .with_max_run_time_minutes(75) .with_script(""" ./mach test-tidy --no-progress --all python3 ./mach test-tidy --no-progress --all --no-wpt python3 ./mach build --dev python3 ./mach test-unit python3 ./mach package --dev python3 ./mach build --dev --features refcell_backtrace python3 ./mach build --dev --features layout-2020 python3 ./mach build --dev --libsimpleservo python3 ./mach build --dev -p servo-gst-plugin python3 ./mach build --dev --media-stack=dummy python3 ./mach test-tidy --no-progress --self-test ./etc/memory_reports_over_time.py --test ./etc/taskcluster/mock.py ./etc/ci/lockfile_changed.sh ./etc/ci/check_no_panic.sh """) .find_or_create("linux_unit." + CONFIG.tree_hash()) )
def macos_unit(): return (macos_build_task("Dev build + unit tests").with_treeherder( "macOS x64", "Unit").with_script(""" ./mach build --dev --verbose ./mach test-unit ./mach package --dev ./etc/ci/lockfile_changed.sh """).find_or_create("macos_unit." + CONFIG.tree_hash()))
def layout_2020_regressions_report(): return (linux_task("Layout 2020 regressions report").with_treeherder( "Linux x64", "RegressionsReport").with_dockerfile( dockerfile_path("base")).with_repo_bundle().with_script( "python3 tests/wpt/reftests-report/gen.py %s %s" % (CONFIG.tree_hash(), CONFIG.git_sha)). with_index_and_artifacts_expire_in(log_artifacts_expire_in). with_artifacts("/repo/tests/wpt/reftests-report/report.html"). with_index_at("layout-2020-regressions-report").create())
def macos_unit(): return (macos_build_task("Dev build + unit tests").with_treeherder( "macOS x64", "Unit").with_script(""" python3 ./mach build --dev --verbose python3 ./mach test-unit python3 ./mach package --dev ./etc/ci/macos_package_smoketest.sh target/debug/servo-tech-demo.dmg ./etc/ci/lockfile_changed.sh """).find_or_create("macos_unit." + CONFIG.tree_hash()))
def linux_release(): return ( linux_build_task("Release build") .with_treeherder("Linux x64", "Release") .with_script( "./mach build --release", "./mach package --release", ) .find_or_create("build.linux_x64_release" + CONFIG.tree_hash()) )
def macos_release_build_with_debug_assertions(priority=None): return (macos_build_task( "Release build, with debug assertions").with_treeherder( "macOS x64", "Release+A" ).with_priority(priority).with_script("\n".join([ "python3 ./mach build --release --verbose --with-debug-assertions", "./etc/ci/lockfile_changed.sh", "tar -czf target.tar.gz" + " target/release/servo" + " target/release/*.dylib" + " resources", ])).with_artifacts("repo/target.tar.gz").find_or_create( "build.macos_x64_release_w_assertions." + CONFIG.tree_hash()))
def macos_nightly(): return (macos_build_task("Nightly build and upload").with_treeherder( "macOS x64", "Nightly").with_features("taskclusterProxy").with_scopes( "secrets:get:project/servo/s3-upload-credentials", "secrets:get:project/servo/github-homebrew-token", ).with_script( "./mach build --release", "./mach package --release", "./mach upload-nightly mac --secret-from-taskcluster", ).with_artifacts("repo/target/release/servo-tech-demo.dmg"). find_or_create("build.mac_x64_nightly." + CONFIG.tree_hash()))
def windows_nightly(rdp=False): return (windows_build_task( "Nightly build and upload", rdp=rdp).with_treeherder( "Windows x64", "Nightly" ).with_features("taskclusterProxy").with_scopes( "secrets:get:project/servo/s3-upload-credentials").with_script( "mach fetch", "mach build --release", "mach package --release", "mach upload-nightly windows-msvc --secret-from-taskcluster"). with_artifacts("repo/target/release/msi/Servo.exe", "repo/target/release/msi/Servo.zip").find_or_create( "build.windows_x64_nightly." + CONFIG.tree_hash()))
def windows_arm64(rdp=False): return (windows_build_task( "UWP dev build", arch="arm64", package=False, rdp=rdp ).with_treeherder( "Windows arm64", "UWP-Dev" ).with_features("taskclusterProxy").with_scopes( "secrets:get:project/servo/windows-codesign-cert/latest" ).with_script( "python mach build --dev --target=aarch64-uwp-windows-msvc", "python mach package --dev --target aarch64-uwp-windows-msvc --uwp=arm64", ).with_artifacts(appx_artifact).find_or_create( "build.windows_uwp_arm64_dev." + CONFIG.tree_hash()))
def linux_nightly(): return ( linux_build_task("Nightly build and upload").with_treeherder( "Linux x64", "Nightly").with_features("taskclusterProxy").with_scopes( "secrets:get:project/servo/s3-upload-credentials") # Not reusing the build made for WPT because it has debug assertions .with_script( "./mach build --release", "./mach package --release", "./mach upload-nightly linux --secret-from-taskcluster", ).with_artifacts("/repo/target/release/servo-tech-demo.tar.gz"). find_or_create("build.linux_x64_nightly" + CONFIG.tree_hash()))
def linux_docs_check(): return ( linux_build_task("Check").with_treeherder( "Linux x64", "Check").with_script( 'RUSTDOCFLAGS="--disable-minification" python3 ./mach doc') # Because `rustdoc` needs metadata of dependency crates, # `cargo doc` does almost all of the work that `cargo check` does. # Therefore, when running them in this order the second command does very little # and should finish quickly. # The reverse order would not increase the total amount of work to do, # but would reduce the amount of parallelism available. .with_script("python3 ./mach check").find_or_create("check." + CONFIG.tree_hash()) )
def macos_release_build_with_debug_assertions(priority=None): return (macos_build_task( "Release build, with debug assertions" ).with_treeherder( "macOS x64", "Release+A" ).with_priority(priority).with_script("\n".join([ "./mach build --release --verbose --with-debug-assertions", "./etc/ci/lockfile_changed.sh", "tar -czf target.tar.gz" + " target/release/servo" + " target/release/build/osmesa-src-*/output" + " target/release/build/osmesa-src-*/out/src/gallium/targets/osmesa/.libs" + " target/release/build/osmesa-src-*/out/src/mapi/shared-glapi/.libs", ])).with_artifacts("repo/target.tar.gz").find_or_create( "build.macos_x64_release_w_assertions." + CONFIG.tree_hash()))
def windows_uwp_x64(): return ( windows_build_task("UWP dev build", package=False) .with_treeherder("Windows x64", "UWP-Dev") .with_features("taskclusterProxy") .with_scopes("secrets:get:project/servo/windows-codesign-cert/latest") .with_script( "python mach build --dev --target=x86_64-uwp-windows-msvc", "python mach package --dev --target=x86_64-uwp-windows-msvc --uwp=x64", "python mach test-tidy --force-cpp --no-wpt", ) .with_artifacts(appx_artifact(debug=True)) .find_or_create("build.windows_uwp_x64_dev." + CONFIG.tree_hash()) )
def uwp_nightly(rdp=False): return (windows_build_task( "Nightly UWP build and upload", package=False, rdp=rdp ).with_treeherder( "Windows x64", "UWP-Nightly" ).with_features("taskclusterProxy").with_scopes( "secrets:get:project/servo/s3-upload-credentials", "secrets:get:project/servo/windows-codesign-cert/latest", ).with_script( "python mach build --release --target=x86_64-uwp-windows-msvc", "python mach build --release --target=aarch64-uwp-windows-msvc", "mach package --release --target=x86_64-uwp-windows-msvc --uwp=x64 --uwp=arm64", "mach upload-nightly uwp --secret-from-taskcluster", ).with_artifacts(appx_artifact).with_max_run_time_minutes( 3 * 60).find_or_create("build.windows_uwp_nightlies." + CONFIG.tree_hash()))
def wpt_chunks(platform, make_chunk_task, build_task, total_chunks, processes, repo_dir, chunks="all", layout_2020=False): if layout_2020: start = 1 # Skip the "extra" WPT testing, a.k.a. chunk 0 name_prefix = "Layout 2020 " job_id_prefix = "2020-" args = "--layout-2020" else: start = 0 name_prefix = "" job_id_prefix = "" args = "" if chunks == "all": chunks = range(start, total_chunks + 1) for this_chunk in chunks: task = ( make_chunk_task("{}WPT chunk {:0{width}} / {}".format( name_prefix, this_chunk, total_chunks, width=len(str(total_chunks)), )) .with_treeherder( platform, "WPT-%s" % this_chunk, group_symbol=job_id_prefix + "WPT", group_name=name_prefix + "web-platform-tests" ) .with_curl_artifact_script(build_task, "target.tar.gz") .with_script("tar -xzf target.tar.gz") .with_index_and_artifacts_expire_in(log_artifacts_expire_in) .with_max_run_time_minutes(90) .with_env( TOTAL_CHUNKS=str(total_chunks), THIS_CHUNK=str(this_chunk), PROCESSES=str(processes), WPT_ARGS=args, GST_DEBUG="3", ) ) # `test-wpt` is piped into `cat` so that stdout is not a TTY # and wptrunner does not use "interactive mode" formatting: # https://github.com/servo/servo/issues/22438 if this_chunk == 0: task.with_script(""" ./mach test-wpt-failure time python2 ./mach test-wpt --release --binary-arg=--multiprocess \ --processes $PROCESSES \ --log-raw test-wpt-mp.log \ --log-errorsummary wpt-mp-errorsummary.log \ eventsource \ | cat time env PYTHONIOENCODING=utf-8 python3 ./mach test-wpt --release --binary-arg=--multiprocess \ --processes $PROCESSES \ --log-raw test-wpt-mp.log \ --log-errorsummary wpt-mp-errorsummary.log \ eventsource \ | cat time ./mach test-wpt --release --product=servodriver --headless \ tests/wpt/mozilla/tests/mozilla/DOMParser.html \ tests/wpt/mozilla/tests/css/per_glyph_font_fallback_a.html \ tests/wpt/mozilla/tests/css/img_simple.html \ tests/wpt/mozilla/tests/mozilla/secure.https.html \ | cat time ./mach test-wpt --release --processes $PROCESSES --product=servodriver \ --headless --log-raw test-bluetooth.log \ --log-errorsummary bluetooth-errorsummary.log \ bluetooth \ | cat time ./mach test-wpt --release --processes $PROCESSES --timeout-multiplier=4 \ --headless --log-raw test-wdspec.log \ --log-servojson wdspec-jsonsummary.log \ --always-succeed \ webdriver \ | cat ./mach filter-intermittents \ wdspec-jsonsummary.log \ --log-intermittents intermittents.log \ --log-filteredsummary filtered-wdspec-errorsummary.log \ --tracker-api default \ --reporter-api default """) else: task.with_script(""" ./mach test-wpt \ --release \ $WPT_ARGS \ --processes $PROCESSES \ --total-chunks "$TOTAL_CHUNKS" \ --this-chunk "$THIS_CHUNK" \ --log-raw test-wpt.log \ --log-servojson wpt-jsonsummary.log \ --always-succeed \ | cat ./mach filter-intermittents \ wpt-jsonsummary.log \ --log-intermittents intermittents.log \ --log-filteredsummary filtered-wpt-errorsummary.log \ --tracker-api default \ --reporter-api default """) task.with_artifacts(*[ "%s/%s" % (repo_dir, word) for script in task.scripts for word in script.split() if word.endswith(".log") ]) task.find_or_create("%s_%swpt_%s.%s" % ( platform.replace(" ", "_").lower(), job_id_prefix.replace("-", "_"), this_chunk, CONFIG.tree_hash(), ))
def wpt_chunks(platform, make_chunk_task, build_task, total_chunks, processes, repo_dir, chunks="all", layout_2020=False, run_webgpu=False): if layout_2020: start = 1 # Skip the "extra" WPT testing, a.k.a. chunk 0 name_prefix = "Layout 2020 " job_id_prefix = "2020-" args = ["--layout-2020"] else: start = 0 name_prefix = "" job_id_prefix = "" args = [] # Our Mac CI runs on machines with an Intel 4000 GPU, so need to work around # https://github.com/servo/webrender/wiki/Driver-issues#bug-1570736---texture-swizzling-affects-wrap-modes-on-some-intel-gpus if platform == "macOS x64": args += ["--pref gfx.texture-swizzling.enabled=false"] if chunks == "all": chunks = range(start, total_chunks + 1) for this_chunk in chunks: task = (make_chunk_task("{}WPT chunk {:0{width}} / {}".format( name_prefix, this_chunk, total_chunks, width=len(str(total_chunks)), )).with_treeherder( platform, "WPT-%s" % this_chunk, group_symbol=job_id_prefix + "WPT", group_name=name_prefix + "web-platform-tests").with_curl_artifact_script( build_task, "target.tar.gz").with_script("tar -xzf target.tar.gz"). with_index_and_artifacts_expire_in(log_artifacts_expire_in). with_max_run_time_minutes(90).with_env( TOTAL_CHUNKS=str(total_chunks), THIS_CHUNK=str(this_chunk), PROCESSES=str(processes), WPT_ARGS=" ".join(args), GST_DEBUG="3", )) # `test-wpt` is piped into `cat` so that stdout is not a TTY # and wptrunner does not use "interactive mode" formatting: # https://github.com/servo/servo/issues/22438 if this_chunk == 0: if run_webgpu: webgpu_script = """ time python3 ./mach test-wpt _webgpu --release --processes $PROCESSES \ --headless --log-raw test-webgpu.log --always-succeed \ --log-errorsummary webgpu-errorsummary.log \ | cat python3 ./mach filter-intermittents \ webgpu-errorsummary.log \ --log-intermittents webgpu-intermittents.log \ --log-filteredsummary filtered-webgpu-errorsummary.log \ --tracker-api default \ --reporter-api default """ # pragma: no cover else: webgpu_script = "" task.with_script(""" time python3 ./mach test-wpt --release --binary-arg=--multiprocess \ --processes $PROCESSES \ --log-raw test-wpt-mp.log \ --log-errorsummary wpt-mp-errorsummary.log \ eventsource \ | cat time env PYTHONIOENCODING=utf-8 python3 ./mach test-wpt --release \ --processes $PROCESSES \ --log-raw test-wpt-py3.log \ --log-errorsummary wpt-py3-errorsummary.log \ --always-succeed \ url \ | cat python3 ./mach filter-intermittents \ wpt-py3-errorsummary.log \ --log-intermittents wpt-py3-intermittents.log \ --log-filteredsummary filtered-py3-errorsummary.log \ --tracker-api default \ --reporter-api default time python3 ./mach test-wpt --release --product=servodriver --headless \ tests/wpt/mozilla/tests/mozilla/DOMParser.html \ tests/wpt/mozilla/tests/css/per_glyph_font_fallback_a.html \ tests/wpt/mozilla/tests/css/img_simple.html \ tests/wpt/mozilla/tests/mozilla/secure.https.html \ | cat time python3 ./mach test-wpt --release --processes $PROCESSES --product=servodriver \ --headless --log-raw test-bluetooth.log \ --log-errorsummary bluetooth-errorsummary.log \ bluetooth \ | cat time python3 ./mach test-wpt --release --processes $PROCESSES --timeout-multiplier=4 \ --headless --log-raw test-wdspec.log \ --log-servojson wdspec-jsonsummary.log \ --always-succeed \ webdriver \ | cat python3 ./mach filter-intermittents \ wdspec-jsonsummary.log \ --log-intermittents intermittents.log \ --log-filteredsummary filtered-wdspec-errorsummary.log \ --tracker-api default \ --reporter-api default """ + webgpu_script) else: task.with_script(""" python3 ./mach test-wpt \ --release \ $WPT_ARGS \ --processes $PROCESSES \ --total-chunks "$TOTAL_CHUNKS" \ --this-chunk "$THIS_CHUNK" \ --log-raw test-wpt.log \ --log-servojson wpt-jsonsummary.log \ --always-succeed \ | cat python3 ./mach filter-intermittents \ wpt-jsonsummary.log \ --log-intermittents intermittents.log \ --log-filteredsummary filtered-wpt-errorsummary.log \ --tracker-api default \ --reporter-api default """) all_artifacts = set([ "%s/%s" % (repo_dir, word) for script in task.scripts for word in script.split() if word.endswith(".log") ]) task.with_artifacts(*all_artifacts) task.find_or_create("%s_%swpt_%s.%s" % ( platform.replace(" ", "_").lower(), job_id_prefix.replace("-", "_"), this_chunk, CONFIG.tree_hash(), ))