def windows_x86(): return ( windows_cross_build_task("Dev build", arch="x86", package=True) .with_treeherder("Windows x86") .with_script( "python mach build --dev --target i686-pc-windows-msvc", "python mach package --dev --target i686-pc-windows-msvc", ) .with_artifacts("repo/target/i686-pc-windows-msvc/debug/msi/Servo.exe", "repo/target/i686-pc-windows-msvc/debug/msi/Servo.zip") .find_or_create("build.windows_x86_dev." + CONFIG.task_id()) )
def windows_uwp_x64(): return (windows_build_task("UWP dev build", package=False).with_treeherder( "Windows x64", "UWP-Dev" ).with_features("taskclusterProxy").with_scopes( "secrets:get:project/servo/windows-codesign-cert/latest" ).with_script( "python mach build --dev --target=x86_64-uwp-windows-msvc", "python mach package --dev --target=x86_64-uwp-windows-msvc --uwp=x64", "python mach test-tidy --force-cpp --no-wpt", ).with_artifacts(appx_artifact( debug=True)).find_or_create("build.windows_uwp_x64_dev." + CONFIG.task_id()))
def android_arm32_dev_from_macos(): return ( macos_build_task("Dev build (macOS)") .with_treeherder("Android ARMv7") .with_script(""" export HOST_CC="$(brew --prefix llvm)/bin/clang" export HOST_CXX="$(brew --prefix llvm)/bin/clang++" ./mach bootstrap-android --accept-all-licences --build ./mach build --android --dev --verbose """) .find_or_create("android_arm32_dev.macos." + CONFIG.task_id()) )
def windows_uwp_x64(): return ( windows_build_task("UWP dev build", package=False) .with_treeherder("Windows x64", "UWP-Dev") .with_script( "python mach build --dev --target=x86_64-uwp-windows-msvc", "python mach package --dev --target=x86_64-uwp-windows-msvc --uwp=x64", "python mach test-tidy --force-cpp --no-wpt", ) .with_artifacts(appx_artifact(debug=True)) .find_or_create("build.windows_uwp_x64_dev." + CONFIG.task_id()) )
def macos_unit(): return ( macos_build_task("Dev build + unit tests") .with_treeherder("macOS x64", "Unit") .with_script(""" ./mach build --dev --verbose ./mach test-unit ./mach package --dev ./etc/ci/lockfile_changed.sh """) .find_or_create("macos_unit." + CONFIG.task_id()) )
def macos_release_build(): return (macos_build_task("Release build").with_treeherder( "macOS x64", "Release").with_script(""" ./mach build --release --verbose ./etc/ci/lockfile_changed.sh tar -czf target.tar.gz \ target/release/servo \ target/release/build/osmesa-src-*/output \ target/release/build/osmesa-src-*/out/src/gallium/targets/osmesa/.libs \ target/release/build/osmesa-src-*/out/src/mapi/shared-glapi/.libs """).with_artifacts("repo/target.tar.gz").find_or_create( "build.macos_x64_release." + CONFIG.task_id()))
def windows_arm64(rdp=False): return (windows_build_task( "UWP dev build", arch="arm64", package=False, rdp=rdp ).with_treeherder( "Windows arm64", "UWP-Dev" ).with_features("taskclusterProxy").with_scopes( "secrets:get:project/servo/windows-codesign-cert/latest" ).with_script( "python mach build --dev --target=aarch64-uwp-windows-msvc", "python mach package --dev --target aarch64-uwp-windows-msvc --uwp=arm64", ).with_artifacts(appx_artifact).find_or_create( "build.windows_uwp_arm64_dev." + CONFIG.tree_hash()))
def macos_release_build_with_debug_assertions(priority=None): return (macos_build_task( "Release build, with debug assertions").with_treeherder( "macOS x64", "Release+A").with_priority(priority).with_script("\n".join([ "./mach build --release --verbose --with-debug-assertions", "./etc/ci/lockfile_changed.sh", "tar -czf target.tar.gz" + " target/release/servo" + " target/release/*.so" + " target/release/*.dylib" + " resources", ])).with_artifacts("repo/target.tar.gz").find_or_create( "build.macos_x64_release_w_assertions." + CONFIG.tree_hash()))
def windows_nightly(rdp=False): return (windows_build_task( "Nightly build and upload", rdp=rdp).with_treeherder( "Windows x64", "Nightly" ).with_features("taskclusterProxy").with_scopes( "secrets:get:project/servo/s3-upload-credentials").with_script( "mach fetch", "mach build --release", "mach package --release", "mach upload-nightly windows-msvc --secret-from-taskcluster"). with_artifacts("repo/target/release/msi/Servo.exe", "repo/target/release/msi/Servo.zip").find_or_create( "build.windows_x64_nightly." + CONFIG.tree_hash()))
def windows_arm64(): return ( windows_cross_build_task("Dev build", arch="arm64", package=False) .with_treeherder("Windows arm64") .with_script( "python mach build --dev --libsimpleservo \ --target aarch64-pc-windows-msvc \ --with-raqote \ --without-wgl", ) .find_or_create("build.windows_arm64_dev." + CONFIG.task_id()) )
def magicleap_nightly(): return ( magicleap_build_task("Nightly build and upload", "Release") .with_features("taskclusterProxy") .with_scopes("secrets:get:project/servo/s3-upload-credentials") .with_script(""" ./mach build --magicleap --release env -u DYLD_LIBRARY_PATH ./mach package --magicleap --release ./mach upload-nightly magicleap --secret-from-taskcluster """) .with_artifacts("repo/target/magicleap/aarch64-linux-android/release/Servo.mpk") .find_or_create("build.magicleap_nightly." + CONFIG.task_id()) )
def linux_nightly(): return ( linux_build_task("Nightly build and upload").with_treeherder( "Linux x64", "Nightly").with_features("taskclusterProxy").with_scopes( "secrets:get:project/servo/s3-upload-credentials") # Not reusing the build made for WPT because it has debug assertions .with_script( "./mach build --release", "./mach package --release", "./mach upload-nightly linux --secret-from-taskcluster", ).with_artifacts("/repo/target/release/servo-tech-demo.tar.gz"). find_or_create("build.linux_x64_nightly" + CONFIG.task_id()))
def macos_nightly(): return (macos_build_task("Nightly build and upload").with_treeherder( "macOS x64", "Nightly" ).with_features("taskclusterProxy").with_scopes( "secrets:get:project/servo/s3-upload-credentials", "secrets:get:project/servo/github-homebrew-token", ).with_script( "./mach build --release", "./mach package --release", "./etc/ci/macos_package_smoketest.sh target/release/servo-tech-demo.dmg", "./mach upload-nightly mac --secret-from-taskcluster", ).with_artifacts("repo/target/release/servo-tech-demo.dmg").find_or_create( "build.mac_x64_nightly." + CONFIG.tree_hash()))
def windows_unit(): return (windows_build_task("Dev build + unit tests").with_treeherder( "Windows x64", "Unit" ).with_script( # Not necessary as this would be done at the start of `build`, # but this allows timing it separately. "mach fetch", "mach build --dev", "mach test-unit", "mach package --dev", ).with_artifacts("repo/target/debug/msi/Servo.exe", "repo/target/debug/msi/Servo.zip").find_or_create( "build.windows_x64_dev." + CONFIG.task_id()))
def macos_unit(): return ( macos_build_task("Dev build + unit tests") .with_treeherder("macOS x64", "Unit") .with_script(""" ./mach build --dev --verbose ./mach test-unit ./mach package --dev ./etc/ci/macos_package_smoketest.sh target/debug/servo-tech-demo.dmg ./etc/ci/lockfile_changed.sh """) .find_or_create("macos_unit." + CONFIG.tree_hash()) )
def macos_release_build_with_debug_assertions(priority=None): return (macos_build_task( "Release build, with debug assertions" ).with_treeherder( "macOS x64", "Release+A" ).with_priority(priority).with_script("\n".join([ "./mach build --release --verbose --with-debug-assertions", "./etc/ci/lockfile_changed.sh", "tar -czf target.tar.gz" + " target/release/servo" + " target/release/build/osmesa-src-*/output" + " target/release/build/osmesa-src-*/out/src/gallium/targets/osmesa/.libs" + " target/release/build/osmesa-src-*/out/src/mapi/shared-glapi/.libs", ])).with_artifacts("repo/target.tar.gz").find_or_create( "build.macos_x64_release_w_assertions." + CONFIG.task_id()))
def linux_docs_check(): return ( linux_build_task("Check").with_treeherder( "Linux x64", "Check").with_script( 'RUSTDOCFLAGS="--disable-minification" python3 ./mach doc') # Because `rustdoc` needs metadata of dependency crates, # `cargo doc` does almost all of the work that `cargo check` does. # Therefore, when running them in this order the second command does very little # and should finish quickly. # The reverse order would not increase the total amount of work to do, # but would reduce the amount of parallelism available. .with_script("python3 ./mach check").find_or_create("check." + CONFIG.tree_hash()) )
def layout_2020_regressions_report(): return ( linux_task("Layout 2020 regressions report") .with_treeherder("Linux x64", "RegressionsReport") .with_dockerfile(dockerfile_path("base")) .with_repo_bundle() .with_script( "python3 etc/layout-2020-regressions/gen.py %s %s" % (CONFIG.tree_hash(), CONFIG.git_sha) ) .with_index_and_artifacts_expire_in(log_artifacts_expire_in) .with_artifacts("/repo/etc/layout-2020-regressions/regressions.html") .find_or_create("layout-2020-regressions-report") )
def windows_x86(): return ( windows_build_task("Dev build", arch="x86") .with_treeherder("Windows x86") .with_env(**{ "VCVARSALL_PATH": "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\BuildTools\\VC\\Auxiliary\\Build" }) .with_script( "python mach build --dev --target i686-pc-windows-msvc", "python mach package --dev --target i686-pc-windows-msvc", ) .with_artifacts("repo/target/i686-pc-windows-msvc/debug/msi/Servo.exe", "repo/target/i686-pc-windows-msvc/debug/msi/Servo.zip") .find_or_create("build.windows_x86_dev." + CONFIG.task_id()) )
def windows_arm64(): return ( windows_build_task("Dev build", arch="arm64", package=False) .with_treeherder("Windows arm64") .with_file_mount( "https://dist.nuget.org/win-x86-commandline/latest/nuget.exe", path="nuget.exe" ) .with_script( "%HOMEDRIVE%%HOMEPATH%\\nuget.exe install ANGLE.WindowsStore.Servo \ -Version 2.1.13 -o %HOMEDRIVE%%HOMEPATH%\\repo\\support\\hololens\\packages", ) .with_script("python mach build --dev --uwp --win-arm64") .find_or_create("build.windows_arm64_dev." + CONFIG.task_id()) )
def daily_tasks_setup(): # ':' is not accepted in an index namepspace: # https://docs.taskcluster.net/docs/reference/core/taskcluster-index/references/api now = SHARED.now.strftime("%Y-%m-%d_%H-%M-%S") index_path = "%s.daily.%s" % (CONFIG.index_prefix, now) # Index this task manually rather than with a route, # so that it is indexed even if it fails. SHARED.index_service.insertTask(index_path, { "taskId": CONFIG.decision_task_id, "rank": 0, "data": {}, "expires": SHARED.from_now_json(log_artifacts_expire_in), }) # Unlike when reacting to a GitHub push event, # the commit hash is not known until we clone the repository. CONFIG.git_sha_is_current_head() # On failure, notify a few people on IRC # https://docs.taskcluster.net/docs/reference/core/taskcluster-notify/docs/usage notify_route = "notify.irc-channel.#servo.on-failed" CONFIG.routes_for_all_subtasks.append(notify_route) CONFIG.scopes_for_all_subtasks.append("queue:route:" + notify_route) CONFIG.task_name_template = "Servo daily: %s. On failure, ping: " + ping_on_daily_task_failure
def uwp_nightly(): return (windows_build_task( "Nightly UWP build and upload", package=False ).with_treeherder( "Windows x64", "UWP Nightly" ).with_features("taskclusterProxy").with_scopes( "secrets:get:project/servo/s3-upload-credentials" ).with_script( "python mach build --release --target=x86_64-uwp-windows-msvc", "python mach build --release --target=aarch64-uwp-windows-msvc", "mach package --release --target=x86_64-uwp-windows-msvc --uwp=x64 --uwp=arm64", "mach upload-nightly uwp --secret-from-taskcluster", ).with_artifacts(appx_artifact(debug=False)).with_max_run_time_minutes( 3 * 60).find_or_create("build.windows_uwp_nightlies." + CONFIG.task_id()))
def daily_tasks_setup(): # ':' is not accepted in an index namepspace: # https://docs.taskcluster.net/docs/reference/core/taskcluster-index/references/api now = SHARED.now.strftime("%Y-%m-%d_%H-%M-%S") index_path = "%s.daily.%s" % (CONFIG.index_prefix, now) # Index this task manually rather than with a route, # so that it is indexed even if it fails. SHARED.index_service.insertTask(index_path, { "taskId": CONFIG.decision_task_id, "rank": 0, "data": {}, "expires": SHARED.from_now_json(log_artifacts_expire_in), }) # Unlike when reacting to a GitHub event, # the commit hash is not known until we clone the repository. CONFIG.git_sha_is_current_head() # On failure, notify a few people on IRC # https://docs.taskcluster.net/docs/reference/core/taskcluster-notify/docs/usage notify_route = "notify.irc-channel.#servo.on-failed" CONFIG.routes_for_all_subtasks.append(notify_route) CONFIG.scopes_for_all_subtasks.append("queue:route:" + notify_route) CONFIG.task_name_template = "Servo daily: %s. On failure, ping: " + ping_on_daily_task_failure
def android_x86_wpt(): build_task = android_x86_release() return (linux_task("WPT").with_treeherder( "Android x86").with_provisioner_id("proj-servo").with_worker_type( "docker-worker-kvm").with_capabilities( privileged=True).with_scopes( "project:servo:docker-worker-kvm:capability:privileged"). with_dockerfile(dockerfile_path("run-android-emulator")).with_repo( ).with_curl_artifact_script( build_task, "servoapp.apk", "target/android/i686-linux-android/release").with_script(""" ./mach bootstrap-android --accept-all-licences --emulator-x86 ./mach test-android-startup --release ./mach test-wpt-android --release \ /_mozilla/mozilla/DOMParser.html \ /_mozilla/mozilla/webgl/context_creation_error.html """).find_or_create("android_x86_release." + CONFIG.task_id()))
def linux_tidy_unit(): return (linux_build_task("Tidy + dev build + unit tests").with_treeherder( "Linux x64", "Tidy+Unit").with_script(""" ./mach test-tidy --no-progress --all ./mach build --dev ./mach test-unit ./mach package --dev ./mach build --dev --features canvas2d-raqote ./mach build --dev --features layout-2020 ./mach build --dev --libsimpleservo ./mach test-tidy --no-progress --self-test ./etc/memory_reports_over_time.py --test ./etc/taskcluster/mock.py ./etc/ci/lockfile_changed.sh ./etc/ci/check_no_panic.sh """).find_or_create("linux_unit." + CONFIG.task_id()))
def windows_unit(worker_type=None, cached=True): task = ( windows_build_task("Dev build + unit tests", worker_type=worker_type). with_treeherder("Windows x64", "Unit").with_script( # Not necessary as this would be done at the start of `build`, # but this allows timing it separately. "mach fetch", "mach build --dev", "mach test-unit", "mach smoketest --angle", "mach package --dev", "mach build --dev --libsimpleservo", ).with_artifacts("repo/target/debug/msi/Servo.exe", "repo/target/debug/msi/Servo.zip")) if cached: return task.find_or_create("build.windows_x64_dev." + CONFIG.task_id()) else: return task.create()
def android_nightly(): return (android_build_task("Nightly build and upload").with_treeherder( "Android Nightlies" ).with_features("taskclusterProxy").with_scopes( "secrets:get:project/servo/s3-upload-credentials" ).with_script(""" ./mach build --release --android ./mach package --release --android --maven ./mach build --release --target i686-linux-android ./mach package --release --target i686-linux-android --maven ./mach upload-nightly android --secret-from-taskcluster ./mach upload-nightly maven --secret-from-taskcluster """).with_artifacts( "/repo/target/android/armv7-linux-androideabi/release/servoapp.apk", "/repo/target/android/armv7-linux-androideabi/release/servoview.aar", "/repo/target/android/i686-linux-android/release/servoapp.apk", "/repo/target/android/i686-linux-android/release/servoview.aar", ).find_or_create("build.android_nightlies." + CONFIG.task_id()))
def linux_tidy_unit(): return (linux_build_task("Tidy + dev build + unit tests").with_treeherder( "Linux x64", "Tidy+Unit").with_max_run_time_minutes(75).with_script(""" python3 ./mach test-tidy --no-progress --all python3 ./mach build --dev python3 ./mach test-unit python3 ./mach package --dev python3 ./mach build --dev --features refcell_backtrace python3 ./mach build --dev --features layout-2020 python3 ./mach build --dev --libsimpleservo python3 ./mach build --dev -p servo-gst-plugin python3 ./mach build --dev --media-stack=dummy python3 ./mach test-tidy --no-progress --self-test ./etc/memory_reports_over_time.py --test ./etc/taskcluster/mock.py ./etc/ci/lockfile_changed.sh ./etc/ci/check_no_panic.sh """).find_or_create("linux_unit." + CONFIG.tree_hash()))
def linux_tidy_unit_docs(): return ( linux_build_task("Tidy + dev build + unit tests + docs") .with_treeherder("Linux x64", "Tidy+Unit+Doc") .with_script(""" ./mach test-tidy --no-progress --all ./mach build --dev ./mach test-unit ./mach package --dev ./mach build --dev --features canvas2d-raqote ./mach build --dev --features layout-2020 ./mach build --dev --libsimpleservo ./mach test-tidy --no-progress --self-test ./etc/memory_reports_over_time.py --test ./etc/taskcluster/mock.py ./etc/ci/lockfile_changed.sh ./etc/ci/check_no_panic.sh rustup component add rust-docs RUSTDOCFLAGS="--disable-minification" ./mach doc ( cd target/doc git init git add . git -c user.name="Taskcluster" -c user.email="" \ commit -q -m "Rebuild Servo documentation" git bundle create docs.bundle HEAD ) """ # Because `rustdoc` needs metadata of dependency crates, # `cargo doc` does almost all of the work that `cargo check` does. # Therefore, when running them in this order the second command does very little # and should finish quickly. # The reverse order would not increase the total amount of work to do, # but would reduce the amount of parallelism available. """ ./mach check """) .with_artifacts("/repo/target/doc/docs.bundle") .find_or_create("docs." + CONFIG.task_id()) )
def linux_wpt(): release_build_task = ( linux_build_task("Release build, with debug assertions") .with_treeherder("Linux x64", "Release+A") .with_script(""" ./mach build --release --with-debug-assertions -p servo ./etc/ci/lockfile_changed.sh tar -czf /target.tar.gz \ target/release/servo \ target/release/build/osmesa-src-*/output \ target/release/build/osmesa-src-*/out/lib/gallium """) .with_artifacts("/target.tar.gz") .find_or_create("build.linux_x64_release~assertions" + CONFIG.task_id()) ) def linux_run_task(name): return linux_task(name).with_dockerfile(dockerfile_path("run")) wpt_chunks("Linux x64", linux_run_task, release_build_task, repo_dir="/repo", total_chunks=2, processes=24)
def upload_docs(): docs_build_task_id = decisionlib.Task.find("docs." + CONFIG.task_id()) return (linux_task("Upload docs to GitHub Pages").with_treeherder( "Linux x64", "DocUpload").with_dockerfile( dockerfile_path("base")).with_curl_artifact_script( docs_build_task_id, "docs.bundle").with_features("taskclusterProxy").with_scopes( "secrets:get:project/servo/doc.servo.org").with_env( PY="""if 1: import urllib, json url = "http://taskcluster/secrets/v1/secret/project/servo/doc.servo.org" token = json.load(urllib.urlopen(url))["secret"]["token"] open("/root/.git-credentials", "w").write("https://*****:*****@github.com/" % token) """).with_script(""" python -c "$PY" git init --bare git config credential.helper store git fetch --quiet docs.bundle git push --force https://github.com/servo/doc.servo.org FETCH_HEAD:gh-pages """).create())
def main(task_for): if CONFIG.git_ref.startswith("refs/heads/"): branch = CONFIG.git_ref[len("refs/heads/"):] CONFIG.treeherder_repository_name = "servo-" + ( branch if not branch.startswith("try-") else "try" ) if task_for == "github-push": # FIXME https://github.com/servo/servo/issues/22325 implement these: linux_arm32_dev = linux_arm64_dev = lambda: None # FIXME https://github.com/servo/servo/issues/22187 # In-emulator testing is disabled for now. (Instead we only compile.) # This local variable shadows the module-level function of the same name. android_x86_wpt = android_x86_release # Implemented but disabled for now: linux_wpt = lambda: None # Shadows the existing top-level function all_tests = [ linux_tidy_unit_docs, windows_unit, windows_x86, macos_unit, magicleap_dev, android_arm32_dev, android_arm32_dev_from_macos, android_arm32_release, android_x86_wpt, linux_arm32_dev, linux_arm64_dev, linux_wpt, macos_wpt, ] by_branch_name = { "auto": all_tests, "try": all_tests, "try-taskcluster": [ # Add functions here as needed, in your push to that branch ], "master": [ upload_docs, ], # The "try-*" keys match those in `servo_try_choosers` in Homu’s config: # https://github.com/servo/saltfs/blob/master/homu/map.jinja "try-mac": [macos_unit], "try-linux": [linux_tidy_unit_docs], "try-windows": [windows_unit, windows_x86], "try-magicleap": [magicleap_dev], "try-arm": [linux_arm32_dev, linux_arm64_dev], "try-wpt": [linux_wpt], "try-wpt-mac": [macos_wpt], "try-wpt-android": [android_x86_wpt], "try-android": [ android_arm32_dev, android_arm32_dev_from_macos, android_x86_wpt ], } for function in by_branch_name.get(branch, []): function() elif task_for == "github-pull-request": CONFIG.treeherder_repository_name = "servo-prs" CONFIG.index_read_only = True CONFIG.docker_image_build_worker_type = None # We want the merge commit that GitHub creates for the PR. # The event does contain a `pull_request.merge_commit_sha` key, but it is wrong: # https://github.com/servo/servo/pull/22597#issuecomment-451518810 CONFIG.git_sha_is_current_head() tidy_untrusted() # https://tools.taskcluster.net/hooks/project-servo/daily elif task_for == "daily": daily_tasks_setup() with_rust_nightly() linux_nightly() android_nightly() windows_nightly() macos_nightly() update_wpt() magicleap_nightly()