def run_cargo(self, params, geckolib=False, check=False): if geckolib: self.set_use_stable_rust() crate_dir = path.join('ports', 'geckolib') else: crate_dir = path.join('components', 'servo') self.ensure_bootstrapped() self.ensure_clobbered() env = self.build_env(geckolib=geckolib) if not params: params = [] if check: params = ['check'] + params build_start = time() if self.context.topdir == getcwd(): with cd(crate_dir): status = call(['cargo'] + params, env=env) else: status = call(['cargo'] + params, env=env) elapsed = time() - build_start notify_build_done(self.config, elapsed, status == 0) if check and status == 0: print('Finished checking, binary NOT updated. Consider ./mach build before ./mach run') return status
def fetch(self): # Fetch Rust and Cargo self.ensure_bootstrapped() # Fetch Cargo dependencies with cd(self.context.topdir): call(["cargo", "fetch"], env=self.build_env())
def cargo(self, params): if not params: params = [] if self.context.topdir == getcwd(): with cd(path.join('components', 'servo')): return call(["cargo"] + params, env=self.build_env()) return call(['cargo'] + params, env=self.build_env())
def fetch(self): # Fetch Rust and Cargo self.ensure_bootstrapped() # Fetch Cargo dependencies for cargo_path in CARGO_PATHS: with cd(cargo_path): print(cargo_path) call(["cargo", "fetch"], env=self.build_env())
def upgrade_wpt_runner(self): with cd(path.join(self.context.topdir, 'tests', 'wpt', 'harness')): code = call(["git", "init"], env=self.build_env()) if code: return code call( ["git", "remote", "add", "upstream", "https://github.com/w3c/wptrunner.git"], env=self.build_env()) code = call(["git", "fetch", "upstream"], env=self.build_env()) if code: return code code = call(["git", "reset", '--', "hard", "remotes/upstream/master"], env=self.build_env()) if code: return code
def build_cef(self, jobs=None, verbose=False, release=False, with_debug_assertions=False): self.ensure_bootstrapped() ret = None opts = [] if jobs is not None: opts += ["-j", jobs] if verbose: opts += ["-v"] if release: opts += ["--release"] servo_features = self.servo_features() if servo_features: opts += ["--features", "%s" % ' '.join(servo_features)] build_start = time() env = self.build_env(is_build=True) if with_debug_assertions: env["RUSTFLAGS"] = "-C debug_assertions" with cd(path.join("ports", "cef")): ret = call(["cargo", "build"] + opts, env=env, verbose=verbose) elapsed = time() - build_start # Generate Desktop Notification if elapsed-time > some threshold value notify_build_done(elapsed) print("CEF build completed in %s" % format_duration(elapsed)) return ret
def run_cargo(self, params, geckolib=False, check=False): if not params: params = [] self.ensure_bootstrapped() self.ensure_clobbered() env = self.build_env(geckolib=geckolib) if check: params = ['check'] + params if geckolib: # for c in $(cargo --list | tail -$(($(cargo --list | wc -l) - 1))); do # (cargo help $c 2>&1 | grep "\\--package" >/dev/null 2>&1) && echo $c # done if params[0] and params[0] in [ 'bench', 'build', 'check', 'clean', 'doc', 'fmt', 'pkgid', 'run', 'rustc', 'rustdoc', 'test', 'update', ]: params[1:1] = ['--package', 'geckoservo'] self.set_use_stable_rust() build_start = time() status = call(['cargo'] + params, env=env) elapsed = time() - build_start notify_build_done(self.config, elapsed, status == 0) if check and status == 0: print('Finished checking, binary NOT updated. Consider ./mach build before ./mach run') return status
def build_geckolib(self, jobs=None, verbose=False, release=False): self.set_use_stable_rust() self.ensure_bootstrapped() self.ensure_clobbered() env = self.build_env(is_build=True, geckolib=True) ret = None opts = ["-p", "geckoservo"] features = [] if jobs is not None: opts += ["-j", jobs] if verbose: opts += ["-v"] if release: opts += ["--release"] if features: opts += ["--features", ' '.join(features)] build_start = time() ret = call(["cargo", "build"] + opts, env=env, verbose=verbose) elapsed = time() - build_start # Generate Desktop Notification if elapsed-time > some threshold value notify_build_done(self.config, elapsed) print("GeckoLib build completed in %s" % format_duration(elapsed)) return ret
def build_geckolib(self, jobs=None, verbose=False, release=False): self.set_use_stable_rust() self.ensure_bootstrapped() ret = None opts = [] if jobs is not None: opts += ["-j", jobs] if verbose: opts += ["-v"] if release: opts += ["--release"] env = self.build_env(is_build=True) env["CARGO_TARGET_DIR"] = path.join(self.context.topdir, "target", "geckolib").encode("UTF-8") build_start = time() with cd(path.join("ports", "geckolib")): ret = call(["cargo", "build"] + opts, env=env, verbose=verbose) elapsed = time() - build_start # Generate Desktop Notification if elapsed-time > some threshold value notify_build_done(elapsed) print("GeckoLib build completed in %s" % format_duration(elapsed)) return ret
def build_geckolib(self, jobs=None, verbose=False, release=False): self.ensure_bootstrapped() ret = None opts = [] if jobs is not None: opts += ["-j", jobs] if verbose: opts += ["-v"] if release: opts += ["--release"] build_start = time() env = self.build_env() with cd(path.join("ports", "geckolib")): ret = call(["cargo", "build"] + opts, env=env, verbose=verbose) elapsed = time() - build_start # Generate Desktop Notification if elapsed-time > some threshold value notify_build_done(elapsed) print("GeckoLib build completed in %s" % format_duration(elapsed)) return ret
def rustc(self, params): if params is None: params = [] self.ensure_bootstrapped() return call(["rustc"] + params, env=self.build_env())
def doc(self, params): self.ensure_bootstrapped() if not path.exists(path.join(self.config["tools"]["rust-root"], "doc")): Registrar.dispatch("bootstrap-rust-docs", context=self.context) rust_docs = path.join(self.config["tools"]["rust-root"], "doc") docs = path.join(self.get_target_dir(), "doc") if not path.exists(docs): os.makedirs(docs) if read_file(path.join(docs, "version_info.html"), if_exists=True) != \ read_file(path.join(rust_docs, "version_info.html")): print("Copying Rust documentation.") # copytree doesn't like the destination already existing. for name in os.listdir(rust_docs): if not name.startswith('.'): full_name = path.join(rust_docs, name) destination = path.join(docs, name) if path.isdir(full_name): if path.exists(destination): rmtree(destination) copytree(full_name, destination) else: copy2(full_name, destination) return call(["cargo", "doc"] + params, env=self.build_env(), cwd=self.servo_crate())
def build_gonk(self, jobs=None, verbose=False, release=False): target = "arm-linux-androideabi" self.ensure_bootstrapped(target=target) opts = [] if jobs is not None: opts += ["-j", jobs] if verbose: opts += ["-v"] if release: opts += ["--release"] opts += ["--target", self.config["android"]["target"]] env = self.build_env(gonk=True) build_start = time() with cd(path.join("ports", "gonk")): ret = call(["cargo", "build"] + opts, env=env, verbose=verbose) elapsed = time() - build_start # Generate Desktop Notification if elapsed-time > some threshold value notify_build_done(elapsed) print("Gonk build completed in %s" % str(datetime.timedelta(seconds=elapsed))) return ret
def test_stylo(self, release=False): self.set_use_stable_rust() self.ensure_bootstrapped() env = self.build_env() env["RUST_BACKTRACE"] = "1" env["CARGO_TARGET_DIR"] = path.join(self.context.topdir, "target", "geckolib").encode("UTF-8") release = ["--release"] if release else [] ret = 0 with cd(path.join("ports", "geckolib")): ret = call(["cargo", "test", "-p", "stylo_tests"] + release, env=env) if ret != 0: return ret with cd(path.join("ports", "geckolib")): return call(["cargo", "test", "-p", "style"] + release, env=env)
def test_geckolib(self): self.ensure_bootstrapped() env = self.build_env() env["RUST_BACKTRACE"] = "1" return call(["cargo", "test"], env=env, cwd=path.join("ports", "geckolib"))
def test_unit(self, test_name=None, package=None): subprocess.check_output([ sys.executable, path.join(self.context.topdir, "components", "style", "list_properties.py") ]) this_file = os.path.dirname(__file__) servo_doc_path = os.path.abspath(os.path.join(this_file, '../', '../', 'target', 'doc', 'servo')) with open(os.path.join(servo_doc_path, 'css-properties.json'), 'r') as property_file: properties = json.loads(property_file.read()) assert len(properties) >= 100 assert "margin-top" in properties assert "margin" in properties if test_name is None: test_name = [] self.ensure_bootstrapped() if package: packages = {package} else: packages = set() test_patterns = [] for test in test_name: # add package if 'tests/unit/<package>' match = re.search("tests/unit/(\\w+)/?$", test) if match: packages.add(match.group(1)) # add package & test if '<package>/<test>', 'tests/unit/<package>/<test>.rs', or similar elif re.search("\\w/\\w", test): tokens = test.split("/") packages.add(tokens[-2]) test_prefix = tokens[-1] if test_prefix.endswith(".rs"): test_prefix = test_prefix[:-3] test_prefix += "::" test_patterns.append(test_prefix) # add test as-is otherwise else: test_patterns.append(test) if not packages: packages = set(os.listdir(path.join(self.context.topdir, "tests", "unit"))) args = ["cargo", "test"] for crate in packages: args += ["-p", "%s_tests" % crate] args += test_patterns env = self.build_env() env["RUST_BACKTRACE"] = "1" result = call(args, env=env, cwd=self.servo_crate()) if result != 0: return result
def rustup(self): url = STATIC_RUST_LANG_ORG_DIST + "/channel-rust-nightly-date.txt" nightly_date = urllib2.urlopen(url, **URLOPEN_KWARGS).read() toolchain = "nightly-" + nightly_date filename = path.join(self.context.topdir, "rust-toolchain") with open(filename, "w") as f: f.write(toolchain + "\n") return call(["rustup" + BIN_SUFFIX, "install", toolchain])
def test_perf(self): self.set_software_rendering_env(True) self.ensure_bootstrapped() env = self.build_env() return call(["bash", "test_perf.sh"], env=env, cwd=path.join("etc", "ci", "performance"))
def rustup(self): url = get_static_rust_lang_org_dist() + "/channel-rust-nightly-date.txt" nightly_date = urllib2.urlopen(url, **get_urlopen_kwargs()).read() toolchain = "nightly-" + nightly_date filename = path.join(self.context.topdir, "rust-toolchain") with open(filename, "w") as f: f.write(toolchain + "\n") return call(["rustup" + BIN_SUFFIX, "install", toolchain])
def test_unit(self, test_name=None, package=None): if test_name is None: test_name = [] self.ensure_bootstrapped() if package: packages = {package} else: packages = set() test_patterns = [] for test in test_name: # add package if 'tests/unit/<package>' match = re.search("tests/unit/(\\w+)/?$", test) if match: packages.add(match.group(1)) # add package & test if '<package>/<test>', 'tests/unit/<package>/<test>.rs', or similar elif re.search("\\w/\\w", test): tokens = test.split("/") packages.add(tokens[-2]) test_prefix = tokens[-1] if test_prefix.endswith(".rs"): test_prefix = test_prefix[:-3] test_prefix += "::" test_patterns.append(test_prefix) # add test as-is otherwise else: test_patterns.append(test) if not packages: packages = set(os.listdir(path.join(self.context.topdir, "tests", "unit"))) packages.remove('stylo') args = ["cargo", "test"] for crate in packages: args += ["-p", "%s_tests" % crate] args += test_patterns features = self.servo_features() if features: args += ["--features", "%s" % ' '.join(features)] env = self.build_env() env["RUST_BACKTRACE"] = "1" if sys.platform in ("win32", "msys"): if "msvc" in host_triple(): # on MSVC, we need some DLLs in the path. They were copied # in to the servo.exe build dir, so just point PATH to that. env["PATH"] = "%s%s%s" % (path.dirname(self.get_binary_path(False, False)), os.pathsep, env["PATH"]) else: env["RUSTFLAGS"] = "-C link-args=-Wl,--subsystem,windows" result = call(args, env=env, cwd=self.servo_crate()) if result != 0: return result
def rustc_geckolib(self, params): if params is None: params = [] self.set_use_stable_rust() self.ensure_bootstrapped() env = self.build_env(geckolib=True) return call(["rustc"] + params, env=env)
def update_cargo(self, params=None, package=None, all_packages=None): if not params: params = [] if package: params += ["-p", package] elif all_packages: params = [] else: print("Please choose package to update with the --package (-p) ") print("flag or update all packages with --all-packages (-a) flag") sys.exit(1) self.ensure_bootstrapped() with cd(self.context.topdir): call(["cargo", "update"] + params, env=self.build_env())
def update_cargo(self, params=None, package=None, all_packages=None): if not params: params = [] if package: params += ["-p", package] elif all_packages: params = [] else: print("Please choose package to update with the --package (-p) ") print("flag or update all packages with --all-packages (-a) flag") sys.exit(1) for cargo_path in CARGO_PATHS: with cd(cargo_path): print(cargo_path) call(["cargo", "update"] + params, env=self.build_env())
def test_wpt_failure(self): self.ensure_bootstrapped() return not call([ "bash", path.join("tests", "wpt", "run.sh"), "--no-pause-after-test", "--include", "infrastructure/failing-test.html" ], env=self.build_env())
def clean(self, manifest_path, params, verbose=False): self.ensure_bootstrapped() opts = [] if manifest_path: opts += ["--manifest-path", manifest_path] if verbose: opts += ["-v"] opts += params return call(["cargo", "clean"] + opts, env=self.build_env(), cwd=self.servo_crate(), verbose=verbose)
def test_stylo(self): self.set_use_stable_rust() self.ensure_bootstrapped() env = self.build_env() env["RUST_BACKTRACE"] = "1" env["CARGO_TARGET_DIR"] = path.join(self.context.topdir, "target", "geckolib").encode("UTF-8") with cd(path.join("ports", "geckolib")): return call(["cargo", "test", "-p", "stylo_tests"], env=env)
def build_tests(self, headless=False, jobs=None, verbose=False, release=False): self.ensure_bootstrapped() args = ["cargo", "test", "--no-run"] if (headless or is_headless_build()) and headless_supported(): args += ["--no-default-features", "--features", "headless"] if release: args += ["--release"] return call( args, env=self.build_env(), cwd=self.servo_crate(), verbose=verbose)
def build_geckolib(self, with_gecko=None, jobs=None, verbose=False, release=False): self.set_use_stable_rust() self.ensure_bootstrapped() env = self.build_env(is_build=True, geckolib=True) geckolib_build_path = path.join(self.context.topdir, "target", "geckolib").encode("UTF-8") ret = None opts = [] features = [] if with_gecko is not None: features += ["bindgen"] env["MOZ_DIST"] = path.abspath(path.expanduser(with_gecko)) if jobs is not None: opts += ["-j", jobs] if verbose: opts += ["-v"] if release: opts += ["--release"] else: features += ["gecko_debug"] if features: opts += ["--features", ' '.join(features)] if with_gecko is not None: print("Generating atoms data...") run_file = path.join(self.context.topdir, "components", "style", "binding_tools", "regen_atoms.py") run_globals = {"__file__": run_file} execfile(run_file, run_globals) run_globals["generate_atoms"](env["MOZ_DIST"]) build_start = time() with cd(path.join("ports", "geckolib")): ret = call(["cargo", "build"] + opts, env=env, verbose=verbose) elapsed = time() - build_start # Generate Desktop Notification if elapsed-time > some threshold value notify_build_done(self.config, elapsed) print("GeckoLib build completed in %s" % format_duration(elapsed)) if with_gecko is not None: print("Copying binding files to style/gecko_bindings...") build_path = path.join(geckolib_build_path, "release" if release else "debug", "") target_style_path = find_dep_path_newest("style", build_path) out_gecko_path = path.join(target_style_path, "out", "gecko") bindings_path = path.join(self.context.topdir, "components", "style", "gecko_bindings") for f in ["bindings.rs", "structs_debug.rs", "structs_release.rs"]: shutil.copy(path.join(out_gecko_path, f), bindings_path) return ret
def test_stylo(self, release=False, test_name=None): self.set_use_stable_rust() self.ensure_bootstrapped() env = self.build_env() env["RUST_BACKTRACE"] = "1" env["CARGO_TARGET_DIR"] = path.join(self.context.topdir, "target", "geckolib").encode("UTF-8") args = (["cargo", "test", "-p", "stylo_tests", "--features", "testing"] + (["--release"] if release else []) + (test_name or [])) with cd(path.join("ports", "geckolib")): return call(args, env=env)
def update_cargo(self, params=None, package=None, all_packages=None): if not params: params = [] if package: params += ["-p", package] elif all_packages: params = [] else: print("Please choose package to update with the --package (-p) ") print("flag or update all packages with --all-packages (-a) flag") sys.exit(1) cargo_paths = [path.join('components', 'servo'), path.join('ports', 'cef'), path.join('ports', 'geckolib')] for cargo_path in cargo_paths: with cd(cargo_path): print(cargo_path) call(["cargo", "update"] + params, env=self.build_env())
def test_perf(self, base=None, date=None, submit=False): self.set_software_rendering_env(True) self.ensure_bootstrapped() env = self.build_env() cmd = ["bash", "test_perf.sh"] if base: cmd += ["--base", base] if date: cmd += ["--date", date] if submit: cmd += ["--submit"] return call(cmd, env=env, cwd=path.join("etc", "ci", "performance"))
def test_stylo(self, release=False, test_name=None): self.set_use_stable_rust() self.ensure_bootstrapped() env = self.build_env() env["RUST_BACKTRACE"] = "1" env["CARGO_TARGET_DIR"] = path.join(self.context.topdir, "target", "geckolib").encode("UTF-8") args = (["cargo", "test", "-p", "stylo_tests"] + (["--release"] if release else []) + (test_name or [])) with cd(path.join("ports", "geckolib")): return call(args, env=env)
def test_compiletest(self, test_name=None, package=None, release=False): if test_name is None: test_name = [] self.ensure_bootstrapped() if package: packages = {package} else: packages = set() test_patterns = [] for test in test_name: # add package if 'tests/compiletest/<package>' match = re.search("tests/compiletest/(\\w+)/?$", test) if match: packages.add(match.group(1)) # add package & test if '<package>/<test>', 'tests/compiletest/<package>/<test>.rs', or similar elif re.search("\\w/\\w", test): tokens = test.split("/") packages.add(tokens[-2]) test_prefix = tokens[-1] if test_prefix.endswith(".rs"): test_prefix = test_prefix[:-3] test_prefix += "::" test_patterns.append(test_prefix) # add test as-is otherwise else: test_patterns.append(test) if not packages: packages = set( os.listdir( path.join(self.context.topdir, "tests", "compiletest"))) - set(['.DS_Store']) packages.remove("helper") args = ["cargo", "test"] for crate in packages: args += ["-p", "%s_compiletest" % crate] args += test_patterns env = self.build_env() if release: env["BUILD_MODE"] = "release" args += ["--release"] else: env["BUILD_MODE"] = "debug" return call(args, env=env, cwd=self.servo_crate())
def test_unit(self, test_name=None, package=None): properties = json.loads( subprocess.check_output([ sys.executable, path.join(self.context.topdir, "components", "style", "list_properties.py") ])) assert len(properties) >= 100 assert "margin-top" in properties assert "margin" in properties if test_name is None: test_name = [] self.ensure_bootstrapped() if package: packages = {package} else: packages = set() test_patterns = [] for test in test_name: # add package if 'tests/unit/<package>' match = re.search("tests/unit/(\\w+)/?$", test) if match: packages.add(match.group(1)) # add package & test if '<package>/<test>', 'tests/unit/<package>/<test>.rs', or similar elif re.search("\\w/\\w", test): tokens = test.split("/") packages.add(tokens[-2]) test_prefix = tokens[-1] if test_prefix.endswith(".rs"): test_prefix = test_prefix[:-3] test_prefix += "::" test_patterns.append(test_prefix) # add test as-is otherwise else: test_patterns.append(test) if not packages: packages = set( os.listdir(path.join(self.context.topdir, "tests", "unit"))) args = ["cargo", "test"] for crate in packages: args += ["-p", "%s_tests" % crate] args += test_patterns result = call(args, env=self.build_env(), cwd=self.servo_crate()) if result != 0: return result
def test_stylo(self): self.set_use_stable_rust() self.ensure_bootstrapped() env = self.build_env() env["RUST_BACKTRACE"] = "1" env["CARGO_TARGET_DIR"] = path.join(self.context.topdir, "target", "geckolib").encode("UTF-8") with cd(path.join("ports", "geckolib")): result = call(["cargo", "test", "-p", "stylo_tests"], env=env) if result != 0: return result
def upgrade_wpt_runner(self): with cd(path.join(self.context.topdir, 'tests', 'wpt', 'harness')): code = call(["git", "init"], env=self.build_env()) if code: return code code = call([ "git", "remote", "add", "upstream", "https://github.com/w3c/wptrunner.git" ], env=self.build_env()) if code: return code code = call(["git", "fetch", "upstream"], env=self.build_env()) if code: return code code = call(["git", "reset", "--hard", "remotes/upstream/master"], env=self.build_env()) if code: return code code = call(["rm", "-rf", ".git"], env=self.build_env()) if code: return code return 0
def update_cargo(self, params=None, package=None, all_packages=None): if not params: params = [] if package: params += ["-p", package] elif all_packages: params = [] else: print("Please choose package to update with the --package (-p) ") print("flag or update all packages with --all-packages (-a) flag") sys.exit(1) cargo_paths = [path.join('components', 'servo'), path.join('ports', 'cef'), path.join('ports', 'geckolib'), path.join('ports', 'gonk')] for cargo_path in cargo_paths: with cd(cargo_path): print(cargo_path) call(["cargo", "update"] + params, env=self.build_env())
def test_unit(self, test_name=None, package=None): if test_name is None: test_name = [] self.ensure_bootstrapped() if package: packages = {package} else: packages = set() test_patterns = [] for test in test_name: # add package if 'tests/unit/<package>' match = re.search("tests/unit/(\\w+)/?$", test) if match: packages.add(match.group(1)) # add package & test if '<package>/<test>', 'tests/unit/<package>/<test>.rs', or similar elif re.search("\\w/\\w", test): tokens = test.split("/") packages.add(tokens[-2]) test_prefix = tokens[-1] if test_prefix.endswith(".rs"): test_prefix = test_prefix[:-3] test_prefix += "::" test_patterns.append(test_prefix) # add test as-is otherwise else: test_patterns.append(test) if not packages: packages = set( os.listdir(path.join(self.context.topdir, "tests", "unit"))) args = ["cargo", "test"] for crate in packages: args += ["-p", "%s_tests" % crate] args += test_patterns features = self.servo_features() if features: args += ["--features", "%s" % ' '.join(features)] env = self.build_env() env["RUST_BACKTRACE"] = "1" result = call(args, env=env, cwd=self.servo_crate()) if result != 0: return result
def build_tests(self, headless=False, jobs=None, verbose=False, release=False): self.ensure_bootstrapped() args = ["cargo", "test", "--no-run"] if (headless or is_headless_build()) and headless_supported(): args += ["--no-default-features", "--features", "headless"] if release: args += ["--release"] return call(args, env=self.build_env(), cwd=self.servo_crate(), verbose=verbose)
def test_perf(self, submit=False): self.set_software_rendering_env(True) self.ensure_bootstrapped() env = self.build_env() cmd = ["bash", "test_perf.sh"] if submit: if not ("TREEHERDER_CLIENT_ID" in os.environ and "TREEHERDER_CLIENT_SECRET" in os.environ): print( "Please set the environment variable \"TREEHERDER_CLIENT_ID\"" " and \"TREEHERDER_CLIENT_SECRET\" to submit the performance" " test result to perfherder") return 1 cmd += ["--submit"] return call(cmd, env=env, cwd=path.join("etc", "ci", "performance"))
def build_cef(self, jobs=None, verbose=False, release=False, with_debug_assertions=False): self.ensure_bootstrapped() self.ensure_clobbered() ret = None opts = [] opts += ["--manifest-path", self.cef_manifest()] if jobs is not None: opts += ["-j", jobs] if verbose: opts += ["-v"] if release: opts += ["--release"] servo_features = self.servo_features() if servo_features: opts += ["--features", "%s" % ' '.join(servo_features)] build_start = time() env = self.build_env(is_build=True) if with_debug_assertions: env["RUSTFLAGS"] = "-C debug_assertions" if is_macosx(): # Unlike RUSTFLAGS, these are only passed in the final rustc invocation # so that `./mach build` followed by `./mach build-cef` both build # common dependencies with the same flags. opts += [ "--", "-C", "link-args=-Xlinker -undefined -Xlinker dynamic_lookup" ] ret = call(["cargo", "rustc"] + opts, env=env, verbose=verbose) elapsed = time() - build_start # Generate Desktop Notification if elapsed-time > some threshold value notify_build_done(self.config, elapsed) print("CEF build completed in %s" % format_duration(elapsed)) return ret
def grep(self, params): if not params: params = [] # get all directories under tests/ tests_dirs = listdir('tests') # Directories to be excluded under tests/ excluded_tests_dirs = ['wpt', 'jquery'] tests_dirs = filter(lambda dir: dir not in excluded_tests_dirs, tests_dirs) # Set of directories in project root root_dirs = ['components', 'ports', 'python', 'etc', 'resources'] # Generate absolute paths for directories in tests/ and project-root/ tests_dirs_abs = [path.join(self.context.topdir, 'tests', s) for s in tests_dirs] root_dirs_abs = [path.join(self.context.topdir, s) for s in root_dirs] # Absolute paths for all directories to be considered grep_paths = root_dirs_abs + tests_dirs_abs return call( ["git"] + ["grep"] + params + ['--'] + grep_paths + [':(exclude)*.min.js', ':(exclude)*.min.css'], env=self.build_env())
def jquery_test_runner(self, cmd, release, dev): base_dir = path.abspath(path.join("tests", "jquery")) jquery_dir = path.join(base_dir, "jquery") run_file = path.join(base_dir, "run_jquery.py") # Clone the jQuery repository if it doesn't exist if not os.path.isdir(jquery_dir): check_call( ["git", "clone", "-b", "servo", "--depth", "1", "https://github.com/servo/jquery", jquery_dir]) # Run pull in case the jQuery repo was updated since last test run check_call( ["git", "-C", jquery_dir, "pull"]) # Check that a release servo build exists bin_path = path.abspath(self.get_binary_path(release, dev)) return call([run_file, cmd, bin_path, base_dir])
def build_geckolib(self, with_gecko=None, jobs=None, verbose=False, release=False): self.set_use_stable_rust() self.ensure_bootstrapped() self.ensure_clobbered() env = self.build_env(is_build=True, geckolib=True) ret = None opts = [] features = [] if with_gecko is not None: features += ["bindgen"] env["MOZ_DIST"] = path.abspath(path.expanduser(with_gecko)) if jobs is not None: opts += ["-j", jobs] if verbose: opts += ["-v"] if release: opts += ["--release"] else: features += ["gecko_debug"] if features: opts += ["--features", ' '.join(features)] build_start = time() with cd(path.join("ports", "geckolib")): ret = call(["cargo", "build"] + opts, env=env, verbose=verbose) elapsed = time() - build_start # Generate Desktop Notification if elapsed-time > some threshold value notify_build_done(self.config, elapsed) print("GeckoLib build completed in %s" % format_duration(elapsed)) return ret
def build_cef(self, jobs=None, verbose=False, release=False, with_debug_assertions=False): self.ensure_bootstrapped() ret = None opts = [] if jobs is not None: opts += ["-j", jobs] if verbose: opts += ["-v"] if release: opts += ["--release"] servo_features = self.servo_features() if servo_features: opts += [ "--features", "%s" % ' '.join("servo/" + x for x in servo_features) ] build_start = time() env = self.build_env(is_build=True) if with_debug_assertions: env["RUSTFLAGS"] = "-C debug_assertions" with cd(path.join("ports", "cef")): ret = call(["cargo", "build"] + opts, env=env, verbose=verbose) elapsed = time() - build_start # Generate Desktop Notification if elapsed-time > some threshold value notify_build_done(elapsed) print("CEF build completed in %s" % format_duration(elapsed)) return ret
def build_cef(self, jobs=None, verbose=False, release=False): self.ensure_bootstrapped() ret = None opts = [] if jobs is not None: opts += ["-j", jobs] if verbose: opts += ["-v"] if release: opts += ["--release"] servo_features = self.servo_features() if servo_features: opts += [ "--features", "%s" % ' '.join("servo/" + x for x in servo_features) ] build_start = time() env = self.build_env(is_build=True) # TODO: If this ends up making it, we should probably add a # --release-with-debug-assertions option or similar, so it's easier to # build locally. if env.get("SERVO_ENABLE_DEBUG_ASSERTIONS", None): env["RUSTFLAGS"] = "-C debug_assertions" with cd(path.join("ports", "cef")): ret = call(["cargo", "build"] + opts, env=env, verbose=verbose) elapsed = time() - build_start # Generate Desktop Notification if elapsed-time > some threshold value notify_build_done(elapsed) print("CEF build completed in %s" % format_duration(elapsed)) return ret
def upgrade_wpt_runner(self): env = self.build_env() with cd(path.join(self.context.topdir, 'tests', 'wpt', 'harness')): code = call(["git", "init"], env=env) if code: return code # No need to report an error if this fails, as it will for the first use call(["git", "remote", "rm", "upstream"], env=env) code = call( ["git", "remote", "add", "upstream", "https://github.com/w3c/wptrunner.git"], env=env) if code: return code code = call(["git", "fetch", "upstream"], env=env) if code: return code code = call(["git", "reset", "--hard", "remotes/upstream/master"], env=env) if code: return code code = call(["rm", "-rf", ".git"], env=env) if code: return code return 0
def build_gonk(self, jobs=None, verbose=False, release=False): self.ensure_bootstrapped() opts = [] if jobs is not None: opts += ["-j", jobs] if verbose: opts += ["-v"] if release: opts += ["--release"] opts += ["--target", self.config["android"]["target"]] env = self.build_env(gonk=True) build_start = time() with cd(path.join("ports", "gonk")): ret = call(["cargo", "build"] + opts, env=env, verbose=verbose) elapsed = time() - build_start # Generate Desktop Notification if elapsed-time > some threshold value notify_build_done(elapsed) print("Gonk build completed in %0.2fs" % elapsed) return ret
def build_geckolib(self, jobs=None, verbose=False, release=False): self.ensure_bootstrapped() ret = None opts = [] if jobs is not None: opts += ["-j", jobs] if verbose: opts += ["-v"] if release: opts += ["--release"] build_start = time() with cd(path.join("ports", "geckolib")): ret = call(["cargo", "build"] + opts, env=self.build_env(), verbose=verbose) elapsed = time() - build_start # Generate Desktop Notification if elapsed-time > some threshold value notify_build_done(elapsed) print("GeckoLib build completed in %s" % str(datetime.timedelta(seconds=elapsed))) return ret
def build(self, target=None, release=False, dev=False, jobs=None, features=None, android=None, verbose=False, debug_mozjs=False, params=None): if android is None: android = self.config["build"]["android"] features = features or [] opts = params or [] base_path = self.get_target_dir() release_path = path.join(base_path, "release", "servo") dev_path = path.join(base_path, "debug", "servo") release_exists = path.exists(release_path) dev_exists = path.exists(dev_path) if not (release or dev): if self.config["build"]["mode"] == "dev": dev = True elif self.config["build"]["mode"] == "release": release = True elif release_exists and not dev_exists: release = True elif dev_exists and not release_exists: dev = True else: print("Please specify either --dev (-d) for a development") print(" build, or --release (-r) for an optimized build.") sys.exit(1) if release and dev: print("Please specify either --dev or --release.") sys.exit(1) if target and android: print("Please specify either --target or --android.") sys.exit(1) if release: opts += ["--release"] if jobs is not None: opts += ["-j", jobs] if verbose: opts += ["-v"] if android: target = self.config["android"]["target"] if target: opts += ["--target", target] self.ensure_bootstrapped(target=target) if debug_mozjs or self.config["build"]["debug-mozjs"]: features += ["script/debugmozjs"] if features: opts += ["--features", "%s" % ' '.join(features)] build_start = time() env = self.build_env(target=target) if android: # Build OpenSSL for android make_cmd = ["make"] if jobs is not None: make_cmd += ["-j" + jobs] android_dir = self.android_build_dir(dev) openssl_dir = path.join(android_dir, "native", "openssl") if not path.exists(openssl_dir): os.makedirs(openssl_dir) shutil.copy( path.join(self.android_support_dir(), "openssl.makefile"), openssl_dir) shutil.copy(path.join(self.android_support_dir(), "openssl.sh"), openssl_dir) with cd(openssl_dir): status = call(make_cmd + ["-f", "openssl.makefile"], env=env, verbose=verbose) if status: return status openssl_dir = path.join(openssl_dir, "openssl-1.0.1t") env['OPENSSL_LIB_DIR'] = openssl_dir env['OPENSSL_INCLUDE_DIR'] = path.join(openssl_dir, "include") env['OPENSSL_STATIC'] = 'TRUE' cargo_binary = "cargo" + BIN_SUFFIX status = call([cargo_binary, "build"] + opts, env=env, cwd=self.servo_crate(), verbose=verbose) elapsed = time() - build_start if sys.platform == "win32" or sys.platform == "msys": shutil.copy( path.join(self.get_top_dir(), "components", "servo", "servo.exe.manifest"), path.join(base_path, "debug" if dev else "release")) # Generate Desktop Notification if elapsed-time > some threshold value notify_build_done(elapsed) print("Build completed in %s" % format_duration(elapsed)) return status
def test_unit(self, test_name=None, package=None): if test_name is None: test_name = [] self.ensure_bootstrapped() if package: packages = {package} else: packages = set() test_patterns = [] for test in test_name: # add package if 'tests/unit/<package>' match = re.search("tests/unit/(\\w+)/?$", test) if match: packages.add(match.group(1)) # add package & test if '<package>/<test>', 'tests/unit/<package>/<test>.rs', or similar elif re.search("\\w/\\w", test): tokens = test.split("/") packages.add(tokens[-2]) test_prefix = tokens[-1] if test_prefix.endswith(".rs"): test_prefix = test_prefix[:-3] test_prefix += "::" test_patterns.append(test_prefix) # add test as-is otherwise else: test_patterns.append(test) if not packages: packages = set( os.listdir(path.join(self.context.topdir, "tests", "unit"))) packages.remove('stylo') args = ["cargo", "test"] for crate in packages: args += ["-p", "%s_tests" % crate] args += test_patterns features = self.servo_features() if features: args += ["--features", "%s" % ' '.join(features)] env = self.build_env() env["RUST_BACKTRACE"] = "1" if sys.platform in ("win32", "msys"): if "msvc" in host_triple(): # on MSVC, we need some DLLs in the path. They were copied # in to the servo.exe build dir, so just point PATH to that. env["PATH"] = "%s%s%s" % (path.dirname( self.get_binary_path(False, False)), os.pathsep, env["PATH"]) else: env["RUSTFLAGS"] = "-C link-args=-Wl,--subsystem,windows" result = call(args, env=env, cwd=self.servo_crate()) if result != 0: return result
def test_unit(self, test_name=None, package=None, bench=False, nocapture=False): if test_name is None: test_name = [] self.ensure_bootstrapped() if package: packages = {package} else: packages = set() test_patterns = [] for test in test_name: # add package if 'tests/unit/<package>' match = re.search("tests/unit/(\\w+)/?$", test) if match: packages.add(match.group(1)) # add package & test if '<package>/<test>', 'tests/unit/<package>/<test>.rs', or similar elif re.search("\\w/\\w", test): tokens = test.split("/") packages.add(tokens[-2]) test_prefix = tokens[-1] if test_prefix.endswith(".rs"): test_prefix = test_prefix[:-3] test_prefix += "::" test_patterns.append(test_prefix) # add test as-is otherwise else: test_patterns.append(test) in_crate_packages = [] # Since the selectors tests have no corresponding selectors_tests crate in tests/unit, # we need to treat them separately from those that do. try: packages.remove('selectors') in_crate_packages += ["selectors"] except KeyError: pass if not packages: packages = set( os.listdir(path.join(self.context.topdir, "tests", "unit"))) - set(['.DS_Store']) in_crate_packages += ["selectors"] # Since the selectors tests have no corresponding selectors_tests crate in tests/unit, # we need to treat them separately from those that do. try: packages.remove('selectors') in_crate_packages += ["selectors"] except KeyError: pass packages.discard('stylo') env = self.build_env() env["RUST_BACKTRACE"] = "1" if "msvc" in host_triple(): # on MSVC, we need some DLLs in the path. They were copied # in to the servo.exe build dir, so just point PATH to that. env["PATH"] = "%s%s%s" % (path.dirname( self.get_binary_path(False, False)), os.pathsep, env["PATH"]) features = self.servo_features() if len(packages) > 0: args = ["cargo", "bench" if bench else "test"] for crate in packages: args += ["-p", "%s_tests" % crate] for crate in in_crate_packages: args += ["-p", crate] args += test_patterns if features: args += ["--features", "%s" % ' '.join(features)] if nocapture: args += ["--", "--nocapture"] err = call(args, env=env, cwd=self.servo_crate()) if err is not 0: return err
def build(self, release=False, dev=False, jobs=None, params=None, media_stack=None, no_package=False, verbose=False, very_verbose=False, target=None, android=False, magicleap=False, libsimpleservo=False, features=None, uwp=False, win_arm64=False, **kwargs): # Force the UWP-enabled target if the convenience UWP flags are passed. if uwp and not target: if win_arm64: target = 'aarch64-uwp-windows-msvc' else: target = 'x86_64-uwp-windows-msvc' opts = params or [] features = features or [] target, android = self.pick_target_triple(target, android, magicleap) # Infer UWP build if only provided a target. if not uwp: uwp = target and 'uwp' in target features += self.pick_media_stack(media_stack, target) target_path = base_path = self.get_target_dir() if android: target_path = path.join(target_path, "android") base_path = path.join(target_path, target) elif magicleap: target_path = path.join(target_path, "magicleap") base_path = path.join(target_path, target) release_path = path.join(base_path, "release", "servo") dev_path = path.join(base_path, "debug", "servo") release_exists = path.exists(release_path) dev_exists = path.exists(dev_path) if not (release or dev): if self.config["build"]["mode"] == "dev": dev = True elif self.config["build"]["mode"] == "release": release = True elif release_exists and not dev_exists: release = True elif dev_exists and not release_exists: dev = True else: print("Please specify either --dev (-d) for a development") print(" build, or --release (-r) for an optimized build.") sys.exit(1) if release and dev: print("Please specify either --dev or --release.") sys.exit(1) if release: opts += ["--release"] servo_path = release_path else: servo_path = dev_path if jobs is not None: opts += ["-j", jobs] if verbose: opts += ["-v"] if very_verbose: opts += ["-vv"] env = self.build_env(target=target, is_build=True, uwp=uwp, features=features) self.ensure_bootstrapped(target=target) self.ensure_clobbered() build_start = time() env["CARGO_TARGET_DIR"] = target_path host = host_triple() target_triple = target or host_triple() if 'apple-darwin' in host and target_triple == host: if 'CXXFLAGS' not in env: env['CXXFLAGS'] = '' env["CXXFLAGS"] += "-mmacosx-version-min=10.10" if 'windows' in host: vs_dirs = self.vs_dirs() if host != target_triple and 'windows' in target_triple: if os.environ.get('VisualStudioVersion') or os.environ.get( 'VCINSTALLDIR'): print( "Can't cross-compile for Windows inside of a Visual Studio shell.\n" "Please run `python mach build [arguments]` to bypass automatic " "Visual Studio shell, and make sure the VisualStudioVersion and " "VCINSTALLDIR environment variables are not set.") sys.exit(1) vcinstalldir = vs_dirs['vcdir'] if not os.path.exists(vcinstalldir): print("Can't find Visual C++ %s installation at %s." % (vs_dirs['vs_version'], vcinstalldir)) sys.exit(1) env['PKG_CONFIG_ALLOW_CROSS'] = "1" if uwp: # Ensure libstd is ready for the new UWP target. check_call(["rustup", "component", "add", "rust-src"]) env['RUST_SYSROOT'] = path.expanduser('~\\.xargo') # Don't try and build a desktop port. libsimpleservo = True arches = { "aarch64": { "angle": "arm64", "gst": "ARM64", "gst_root": "arm64", }, "x86_64": { "angle": "x64", "gst": "X86_64", "gst_root": "x64", }, } arch = arches.get(target_triple.split('-')[0]) if not arch: print("Unsupported UWP target.") sys.exit(1) # Ensure that the NuGet ANGLE package containing libEGL is accessible # to the Rust linker. append_to_path_env(angle_root(target_triple, env), env, "LIB") # Don't want to mix non-UWP libraries with vendored UWP libraries. if "gstreamer" in env['LIB']: print( "Found existing GStreamer library path in LIB. Please remove it." ) sys.exit(1) # Override any existing GStreamer installation with the vendored libraries. env["GSTREAMER_1_0_ROOT_" + arch['gst']] = path.join( self.msvc_package_dir("gstreamer-uwp"), arch['gst_root']) env["PKG_CONFIG_PATH"] = path.join( self.msvc_package_dir("gstreamer-uwp"), arch['gst_root'], "lib", "pkgconfig") if 'windows' in host: process = subprocess.Popen( '("%s" %s > nul) && "python" -c "import os; print(repr(os.environ))"' % (os.path.join(vs_dirs['vcdir'], "Auxiliary", "Build", "vcvarsall.bat"), "x64"), stdout=subprocess.PIPE, shell=True) stdout, stderr = process.communicate() exitcode = process.wait() encoding = locale.getpreferredencoding( ) # See https://stackoverflow.com/a/9228117 if exitcode == 0: decoded = stdout.decode(encoding) if decoded.startswith("environ("): decoded = decoded.strip()[8:-1] os.environ.update(eval(decoded)) else: print("Failed to run vcvarsall. stderr:") print(stderr.decode(encoding)) exit(1) # Ensure that GStreamer libraries are accessible when linking. if 'windows' in target_triple: gst_root = gstreamer_root(target_triple, env) if gst_root: append_to_path_env(os.path.join(gst_root, "lib"), env, "LIB") if android: if "ANDROID_NDK" not in env: print("Please set the ANDROID_NDK environment variable.") sys.exit(1) if "ANDROID_SDK" not in env: print("Please set the ANDROID_SDK environment variable.") sys.exit(1) android_platform = self.config["android"]["platform"] android_toolchain_name = self.config["android"]["toolchain_name"] android_toolchain_prefix = self.config["android"][ "toolchain_prefix"] android_lib = self.config["android"]["lib"] android_arch = self.config["android"]["arch"] # Build OpenSSL for android env["OPENSSL_VERSION"] = "1.1.1d" make_cmd = ["make"] if jobs is not None: make_cmd += ["-j" + jobs] openssl_dir = path.join(target_path, target, "native", "openssl") if not path.exists(openssl_dir): os.makedirs(openssl_dir) shutil.copy( path.join(self.android_support_dir(), "openssl.makefile"), openssl_dir) shutil.copy(path.join(self.android_support_dir(), "openssl.sh"), openssl_dir) # Check if the NDK version is 15 if not os.path.isfile( path.join(env["ANDROID_NDK"], 'source.properties')): print("ANDROID_NDK should have file `source.properties`.") print( "The environment variable ANDROID_NDK may be set at a wrong path." ) sys.exit(1) with open(path.join(env["ANDROID_NDK"], 'source.properties')) as ndk_properties: lines = ndk_properties.readlines() if lines[1].split(' = ')[1].split('.')[0] != '15': print( "Currently only support NDK 15. Please re-run `./mach bootstrap-android`." ) sys.exit(1) env["RUST_TARGET"] = target with cd(openssl_dir): status = call(make_cmd + ["-f", "openssl.makefile"], env=env, verbose=verbose) if status: return status openssl_dir = path.join( openssl_dir, "openssl-{}".format(env["OPENSSL_VERSION"])) env['OPENSSL_LIB_DIR'] = openssl_dir env['OPENSSL_INCLUDE_DIR'] = path.join(openssl_dir, "include") env['OPENSSL_STATIC'] = 'TRUE' # Android builds also require having the gcc bits on the PATH and various INCLUDE # path munging if you do not want to install a standalone NDK. See: # https://dxr.mozilla.org/mozilla-central/source/build/autoconf/android.m4#139-161 os_type = platform.system().lower() if os_type not in ["linux", "darwin"]: raise Exception( "Android cross builds are only supported on Linux and macOS." ) cpu_type = platform.machine().lower() host_suffix = "unknown" if cpu_type in ["i386", "i486", "i686", "i768", "x86"]: host_suffix = "x86" elif cpu_type in ["x86_64", "x86-64", "x64", "amd64"]: host_suffix = "x86_64" host = os_type + "-" + host_suffix host_cc = env.get('HOST_CC') or _get_exec_path( ["clang"]) or _get_exec_path(["gcc"]) host_cxx = env.get('HOST_CXX') or _get_exec_path( ["clang++"]) or _get_exec_path(["g++"]) llvm_toolchain = path.join(env['ANDROID_NDK'], "toolchains", "llvm", "prebuilt", host) gcc_toolchain = path.join(env['ANDROID_NDK'], "toolchains", android_toolchain_prefix + "-4.9", "prebuilt", host) gcc_libs = path.join(gcc_toolchain, "lib", "gcc", android_toolchain_name, "4.9.x") env['PATH'] = (path.join(llvm_toolchain, "bin") + ':' + env['PATH']) env['ANDROID_SYSROOT'] = path.join(env['ANDROID_NDK'], "sysroot") support_include = path.join(env['ANDROID_NDK'], "sources", "android", "support", "include") cpufeatures_include = path.join(env['ANDROID_NDK'], "sources", "android", "cpufeatures") cxx_include = path.join(env['ANDROID_NDK'], "sources", "cxx-stl", "llvm-libc++", "include") clang_include = path.join(llvm_toolchain, "lib64", "clang", "3.8", "include") cxxabi_include = path.join(env['ANDROID_NDK'], "sources", "cxx-stl", "llvm-libc++abi", "include") sysroot_include = path.join(env['ANDROID_SYSROOT'], "usr", "include") arch_include = path.join(sysroot_include, android_toolchain_name) android_platform_dir = path.join(env['ANDROID_NDK'], "platforms", android_platform, "arch-" + android_arch) arch_libs = path.join(android_platform_dir, "usr", "lib") clang_include = path.join(llvm_toolchain, "lib64", "clang", "5.0", "include") android_api = android_platform.replace('android-', '') env['HOST_CC'] = host_cc env['HOST_CXX'] = host_cxx env['HOST_CFLAGS'] = '' env['HOST_CXXFLAGS'] = '' env['CC'] = path.join(llvm_toolchain, "bin", "clang") env['CPP'] = path.join(llvm_toolchain, "bin", "clang") + " -E" env['CXX'] = path.join(llvm_toolchain, "bin", "clang++") env['ANDROID_TOOLCHAIN'] = gcc_toolchain env['ANDROID_TOOLCHAIN_DIR'] = gcc_toolchain env['ANDROID_VERSION'] = android_api env['ANDROID_PLATFORM_DIR'] = android_platform_dir env['GCC_TOOLCHAIN'] = gcc_toolchain gcc_toolchain_bin = path.join(gcc_toolchain, android_toolchain_name, "bin") env['AR'] = path.join(gcc_toolchain_bin, "ar") env['RANLIB'] = path.join(gcc_toolchain_bin, "ranlib") env['OBJCOPY'] = path.join(gcc_toolchain_bin, "objcopy") env['YASM'] = path.join(env['ANDROID_NDK'], 'prebuilt', host, 'bin', 'yasm') # A cheat-sheet for some of the build errors caused by getting the search path wrong... # # fatal error: 'limits' file not found # -- add -I cxx_include # unknown type name '__locale_t' (when running bindgen in mozjs_sys) # -- add -isystem sysroot_include # error: use of undeclared identifier 'UINTMAX_C' # -- add -D__STDC_CONSTANT_MACROS # # Also worth remembering: autoconf uses C for its configuration, # even for C++ builds, so the C flags need to line up with the C++ flags. env['CFLAGS'] = ' '.join([ "--target=" + target, "--sysroot=" + env['ANDROID_SYSROOT'], "--gcc-toolchain=" + gcc_toolchain, "-isystem", sysroot_include, "-I" + arch_include, "-B" + arch_libs, "-L" + arch_libs, "-D__ANDROID_API__=" + android_api, ]) env['CXXFLAGS'] = ' '.join([ "--target=" + target, "--sysroot=" + env['ANDROID_SYSROOT'], "--gcc-toolchain=" + gcc_toolchain, "-I" + cpufeatures_include, "-I" + cxx_include, "-I" + clang_include, "-isystem", sysroot_include, "-I" + cxxabi_include, "-I" + clang_include, "-I" + arch_include, "-I" + support_include, "-L" + gcc_libs, "-B" + arch_libs, "-L" + arch_libs, "-D__ANDROID_API__=" + android_api, "-D__STDC_CONSTANT_MACROS", "-D__NDK_FPABI__=", ]) env['CPPFLAGS'] = ' '.join([ "--target=" + target, "--sysroot=" + env['ANDROID_SYSROOT'], "-I" + arch_include, ]) env["NDK_ANDROID_VERSION"] = android_api env["ANDROID_ABI"] = android_lib env["ANDROID_PLATFORM"] = android_platform env["NDK_CMAKE_TOOLCHAIN_FILE"] = path.join( env['ANDROID_NDK'], "build", "cmake", "android.toolchain.cmake") env["CMAKE_TOOLCHAIN_FILE"] = path.join(self.android_support_dir(), "toolchain.cmake") # Set output dir for gradle aar files aar_out_dir = self.android_aar_dir() if not os.path.exists(aar_out_dir): os.makedirs(aar_out_dir) env["AAR_OUT_DIR"] = aar_out_dir # GStreamer and its dependencies use pkg-config and this flag is required # to make it work in a cross-compilation context. env["PKG_CONFIG_ALLOW_CROSS"] = '1' # Build the name of the package containing all GStreamer dependencies # according to the build target. gst_lib = "gst-build-{}".format(self.config["android"]["lib"]) gst_lib_zip = "gstreamer-{}-1.16.0-20190517-095630.zip".format( self.config["android"]["lib"]) gst_dir = os.path.join(target_path, "gstreamer") gst_lib_path = os.path.join(gst_dir, gst_lib) pkg_config_path = os.path.join(gst_lib_path, "pkgconfig") env["PKG_CONFIG_PATH"] = pkg_config_path if not os.path.exists(gst_lib_path): # Download GStreamer dependencies if they have not already been downloaded # This bundle is generated with `libgstreamer_android_gen` # Follow these instructions to build and deploy new binaries # https://github.com/servo/libgstreamer_android_gen#build print("Downloading GStreamer dependencies") gst_url = "https://servo-deps.s3.amazonaws.com/gstreamer/%s" % gst_lib_zip print(gst_url) urllib.request.urlretrieve(gst_url, gst_lib_zip) zip_ref = zipfile.ZipFile(gst_lib_zip, "r") zip_ref.extractall(gst_dir) os.remove(gst_lib_zip) # Change pkgconfig info to make all GStreamer dependencies point # to the libgstreamer_android.so bundle. for each in os.listdir(pkg_config_path): if each.endswith('.pc'): print("Setting pkgconfig info for %s" % each) pc = os.path.join(pkg_config_path, each) expr = "s#libdir=.*#libdir=%s#g" % gst_lib_path subprocess.call(["perl", "-i", "-pe", expr, pc]) if magicleap: if platform.system() not in ["Darwin"]: raise Exception( "Magic Leap builds are only supported on macOS. " "If you only wish to test if your code builds, " "run ./mach build -p libmlservo.") ml_sdk = env.get("MAGICLEAP_SDK") if not ml_sdk: raise Exception( "Magic Leap builds need the MAGICLEAP_SDK environment variable" ) if not os.path.exists(ml_sdk): raise Exception( "Path specified by MAGICLEAP_SDK does not exist.") ml_support = path.join(self.get_top_dir(), "support", "magicleap") # We pretend to be an Android build env.setdefault("ANDROID_VERSION", "21") env.setdefault("ANDROID_NDK", env["MAGICLEAP_SDK"]) env.setdefault("ANDROID_NDK_VERSION", "16.0.0") env.setdefault("ANDROID_PLATFORM_DIR", path.join(env["MAGICLEAP_SDK"], "lumin")) env.setdefault( "ANDROID_TOOLCHAIN_DIR", path.join(env["MAGICLEAP_SDK"], "tools", "toolchains")) env.setdefault( "ANDROID_CLANG", path.join(env["ANDROID_TOOLCHAIN_DIR"], "bin", "clang")) # A random collection of search paths env.setdefault( "STLPORT_LIBS", " ".join([ "-L" + path.join(env["MAGICLEAP_SDK"], "lumin", "stl", "libc++-lumin", "lib"), "-lc++" ])) env.setdefault( "STLPORT_CPPFLAGS", " ".join([ "-I" + path.join(env["MAGICLEAP_SDK"], "lumin", "stl", "libc++-lumin", "include") ])) env.setdefault( "CPPFLAGS", " ".join([ "--no-standard-includes", "--sysroot=" + env["ANDROID_PLATFORM_DIR"], "-I" + path.join(env["ANDROID_PLATFORM_DIR"], "usr", "include"), "-isystem" + path.join(env["ANDROID_TOOLCHAIN_DIR"], "lib64", "clang", "3.8", "include"), ])) env.setdefault( "CFLAGS", " ".join([ env["CPPFLAGS"], "-L" + path.join(env["ANDROID_TOOLCHAIN_DIR"], "lib", "gcc", target, "4.9.x"), ])) env.setdefault( "CXXFLAGS", " ".join([ # Sigh, Angle gets confused if there's another EGL around "-I./gfx/angle/checkout/include", env["STLPORT_CPPFLAGS"], env["CFLAGS"] ])) # The toolchain commands env.setdefault( "AR", path.join(env["ANDROID_TOOLCHAIN_DIR"], "bin", "aarch64-linux-android-ar")) env.setdefault( "AS", path.join(env["ANDROID_TOOLCHAIN_DIR"], "bin", "aarch64-linux-android-clang")) env.setdefault( "CC", path.join(env["ANDROID_TOOLCHAIN_DIR"], "bin", "aarch64-linux-android-clang")) env.setdefault( "CPP", path.join(env["ANDROID_TOOLCHAIN_DIR"], "bin", "aarch64-linux-android-clang -E")) env.setdefault( "CXX", path.join(env["ANDROID_TOOLCHAIN_DIR"], "bin", "aarch64-linux-android-clang++")) env.setdefault( "LD", path.join(env["ANDROID_TOOLCHAIN_DIR"], "bin", "aarch64-linux-android-ld")) env.setdefault( "OBJCOPY", path.join(env["ANDROID_TOOLCHAIN_DIR"], "bin", "aarch64-linux-android-objcopy")) env.setdefault( "OBJDUMP", path.join(env["ANDROID_TOOLCHAIN_DIR"], "bin", "aarch64-linux-android-objdump")) env.setdefault( "RANLIB", path.join(env["ANDROID_TOOLCHAIN_DIR"], "bin", "aarch64-linux-android-ranlib")) env.setdefault( "STRIP", path.join(env["ANDROID_TOOLCHAIN_DIR"], "bin", "aarch64-linux-android-strip")) # Undo all of that when compiling build tools for the host env.setdefault("HOST_CFLAGS", "") env.setdefault("HOST_CXXFLAGS", "") env.setdefault("HOST_CC", "/usr/local/opt/llvm/bin/clang") env.setdefault("HOST_CXX", "/usr/local/opt/llvm/bin/clang++") env.setdefault("HOST_LD", "ld") # Some random build configurations env.setdefault("HARFBUZZ_SYS_NO_PKG_CONFIG", "1") env.setdefault("PKG_CONFIG_ALLOW_CROSS", "1") env.setdefault("CMAKE_TOOLCHAIN_FILE", path.join(ml_support, "toolchain.cmake")) env.setdefault("_LIBCPP_INLINE_VISIBILITY", "__attribute__((__always_inline__))") # The Open SSL configuration env.setdefault("OPENSSL_DIR", path.join(target_path, target, "native", "openssl")) env.setdefault("OPENSSL_VERSION", "1.1.1d") env.setdefault("OPENSSL_STATIC", "1") # GStreamer configuration env.setdefault( "GSTREAMER_DIR", path.join(target_path, target, "native", "gstreamer-1.16.0")) env.setdefault( "GSTREAMER_URL", "https://servo-deps.s3.amazonaws.com/gstreamer/gstreamer-magicleap-1.16.0-20190823-104505.tgz" ) env.setdefault( "PKG_CONFIG_PATH", path.join(env["GSTREAMER_DIR"], "system", "lib64", "pkgconfig")) # Override the linker set in .cargo/config env.setdefault("CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER", path.join(ml_support, "fake-ld.sh")) # Only build libmlservo opts += ["--package", "libmlservo"] # Download and build OpenSSL if necessary status = call(path.join(ml_support, "openssl.sh"), env=env, verbose=verbose) if status: return status # Download prebuilt Gstreamer if necessary if not os.path.exists(path.join(env["GSTREAMER_DIR"], "system")): if not os.path.exists(env["GSTREAMER_DIR"] + ".tgz"): check_call([ 'curl', '-L', '-f', '-o', env["GSTREAMER_DIR"] + ".tgz", env["GSTREAMER_URL"], ]) check_call([ 'mkdir', '-p', env["GSTREAMER_DIR"], ]) check_call([ 'tar', 'xzf', env["GSTREAMER_DIR"] + ".tgz", '-C', env["GSTREAMER_DIR"], ]) # https://internals.rust-lang.org/t/exploring-crate-graph-build-times-with-cargo-build-ztimings/10975 # Prepend so that e.g. `-Ztimings` (which means `-Ztimings=info,html`) # given on the command line can override it opts = ["-Ztimings=info"] + opts if very_verbose: print(["Calling", "cargo", "build"] + opts) for key in env: print((key, env[key])) if sys.platform == "win32": env.setdefault("CC", "clang-cl.exe") env.setdefault("CXX", "clang-cl.exe") if uwp: env.setdefault("CFLAGS", "") env.setdefault("CXXFLAGS", "") env["CFLAGS"] += " -DWINAPI_FAMILY=WINAPI_FAMILY_APP" env["CXXFLAGS"] += " -DWINAPI_FAMILY=WINAPI_FAMILY_APP" else: env.setdefault("CC", "clang") env.setdefault("CXX", "clang++") status = self.run_cargo_build_like_command( "build", opts, env=env, verbose=verbose, target=target, android=android, magicleap=magicleap, libsimpleservo=libsimpleservo, uwp=uwp, features=features, **kwargs) elapsed = time() - build_start # Do some additional things if the build succeeded if status == 0: if android and not no_package: flavor = None if "googlevr" in features: flavor = "googlevr" elif "oculusvr" in features: flavor = "oculusvr" rv = Registrar.dispatch("package", context=self.context, release=release, dev=dev, target=target, flavor=flavor) if rv: return rv if sys.platform == "win32": servo_exe_dir = os.path.dirname( self.get_binary_path(release, dev, target=target, simpleservo=libsimpleservo)) assert os.path.exists(servo_exe_dir) # on msvc builds, use editbin to change the subsystem to windows, but only # on release builds -- on debug builds, it hides log output if not dev and not libsimpleservo: call([ "editbin", "/nologo", "/subsystem:windows", path.join(servo_exe_dir, "servo.exe") ], verbose=verbose) # on msvc, we need to copy in some DLLs in to the servo.exe dir and the directory for unit tests. for ssl_lib in ["libssl.dll", "libcrypto.dll"]: ssl_path = path.join(env['OPENSSL_LIB_DIR'], "../bin", ssl_lib) shutil.copy(ssl_path, servo_exe_dir) shutil.copy(ssl_path, path.join(servo_exe_dir, "deps")) build_path = path.join(servo_exe_dir, "build") assert os.path.exists(build_path) def package_generated_shared_libraries(libs, build_path, servo_exe_dir): for root, dirs, files in os.walk(build_path): remaining_libs = list(libs) for lib in libs: if lib in files: shutil.copy(path.join(root, lib), servo_exe_dir) remaining_libs.remove(lib) continue libs = remaining_libs if not libs: return True for lib in libs: print("WARNING: could not find " + lib) # UWP build has its own ANGLE library that it packages. if not uwp: print("Packaging EGL DLLs") egl_libs = ["libEGL.dll", "libGLESv2.dll"] if not package_generated_shared_libraries( egl_libs, build_path, servo_exe_dir): status = 1 # copy needed gstreamer DLLs in to servo.exe dir print("Packaging gstreamer DLLs") if not package_gstreamer_dlls(env, servo_exe_dir, target_triple, uwp): status = 1 # UWP app packaging already bundles all required DLLs for us. print("Packaging MSVC DLLs") if not package_msvc_dlls(servo_exe_dir, target_triple, vs_dirs['vcdir'], vs_dirs['vs_version']): status = 1 elif sys.platform == "darwin": # On the Mac, set a lovely icon. This makes it easier to pick out the Servo binary in tools # like Instruments.app. try: import Cocoa icon_path = path.join(self.get_top_dir(), "resources", "servo_1024.png") icon = Cocoa.NSImage.alloc().initWithContentsOfFile_( icon_path) if icon is not None: Cocoa.NSWorkspace.sharedWorkspace( ).setIcon_forFile_options_(icon, servo_path, 0) except ImportError: pass # Generate Desktop Notification if elapsed-time > some threshold value notify_build_done(self.config, elapsed, status == 0) print("Build %s in %s" % ("Completed" if status == 0 else "FAILED", format_duration(elapsed))) return status
def notify(title, text): if call([command, title, text]) != 0: raise Exception("Could not run '%s'." % command)
def build(self, target=None, release=False, dev=False, jobs=None, features=None, android=None, verbose=False, debug_mozjs=False, params=None): if android is None: android = self.config["build"]["android"] features = features or [] opts = params or [] base_path = self.get_target_dir() release_path = path.join(base_path, "release", "servo") dev_path = path.join(base_path, "debug", "servo") release_exists = path.exists(release_path) dev_exists = path.exists(dev_path) if not (release or dev): if self.config["build"]["mode"] == "dev": dev = True elif self.config["build"]["mode"] == "release": release = True elif release_exists and not dev_exists: release = True elif dev_exists and not release_exists: dev = True else: print("Please specify either --dev (-d) for a development") print(" build, or --release (-r) for an optimized build.") sys.exit(1) if release and dev: print("Please specify either --dev or --release.") sys.exit(1) targets = [] if release: opts += ["--release"] if target: opts += ["--target", target] targets.append(target) if jobs is not None: opts += ["-j", jobs] if verbose: opts += ["-v"] if android: opts += ["--target", self.config["android"]["target"]] targets.append("arm-linux-androideabi") self.ensure_bootstrapped(targets=targets) if debug_mozjs or self.config["build"]["debug-mozjs"]: features += ["script/debugmozjs"] if android: features += ["android_glue"] if features: opts += ["--features", "%s" % ' '.join(features)] build_start = time() env = self.build_env() if android: # Ensure Rust uses hard floats on Android env['RUSTFLAGS'] = env.get('RUSTFLAGS', "") + " -C target-feature=+neon" # Build OpenSSL for android make_cmd = ["make"] if jobs is not None: make_cmd += ["-j" + jobs] android_dir = self.android_build_dir(dev) openssl_dir = path.join(android_dir, "native", "openssl") if not path.exists(openssl_dir): os.makedirs(openssl_dir) shutil.copy( path.join(self.android_support_dir(), "openssl.makefile"), openssl_dir) shutil.copy(path.join(self.android_support_dir(), "openssl.sh"), openssl_dir) with cd(openssl_dir): status = call(make_cmd + ["-f", "openssl.makefile"], env=self.build_env(), verbose=verbose) if status: return status openssl_dir = path.join(openssl_dir, "openssl-1.0.1k") env['OPENSSL_LIB_DIR'] = openssl_dir env['OPENSSL_INCLUDE_DIR'] = path.join(openssl_dir, "include") env['OPENSSL_STATIC'] = 'TRUE' if not (self.config["build"]["ccache"] == ""): env['CCACHE'] = self.config["build"]["ccache"] env['RUSTFLAGS'] = env.get('RUSTFLAGS', "") + " -W unused-extern-crates" status = call(["cargo", "build"] + opts, env=env, cwd=self.servo_crate(), verbose=verbose) elapsed = time() - build_start # Generate Desktop Notification if elapsed-time > some threshold value notify_build_done(elapsed) print("Build completed in %0.2fs" % elapsed) return status
def update_cargo(self, params=None, package=None, all_packages=None, dry_run=None): if not params: params = [] if dry_run: import toml import json import httplib import colorama cargo_file = open(path.join(self.context.topdir, "Cargo.lock")) content = toml.load(cargo_file) packages = {} outdated_packages = 0 conn = httplib.HTTPSConnection("crates.io") for package in content.get("package", []): if "replace" in package: continue source = package.get("source", "") if source == r"registry+https://github.com/rust-lang/crates.io-index": version = package["version"] name = package["name"] if not packages.get(name, "") or packages[name] > version: packages[name] = package["version"] conn.request( 'GET', '/api/v1/crates/{}/versions'.format( package["name"])) r = conn.getresponse() json_content = json.load(r) for v in json_content.get("versions"): if not v.get("yanked"): max_version = v.get("num") break if version != max_version: outdated_packages += 1 version_major, version_minor = ( version.split("."))[:2] max_major, max_minor = (max_version.split("."))[:2] if version_major == max_major and version_minor == max_minor and "alpha" not in version: msg = "minor update" msg_color = "\033[93m" else: msg = "update, which may contain breaking changes" msg_color = "\033[91m" colorama.init() print( "{}Outdated package `{}`, available {}\033[0m". format(msg_color, name, msg), "\n\tCurrent version: {}".format(version), "\n\t Latest version: {}".format(max_version)) conn.close() print("\nFound {} outdated packages from crates.io".format( outdated_packages)) elif package: params += ["-p", package] elif all_packages: params = [] else: print("Please choose package to update with the --package (-p) ") print("flag or update all packages with --all-packages (-a) flag") sys.exit(1) if params or all_packages: self.ensure_bootstrapped() with cd(self.context.topdir): call(["cargo", "update"] + params, env=self.build_env())
def rustc(self, params): if params is None: params = [] return call(["rustc"] + params, env=self.build_env())
def build(self, target=None, release=False, dev=False, jobs=None, features=None, android=None, verbose=False, debug_mozjs=False, params=None, with_debug_assertions=False): if android is None: android = self.config["build"]["android"] features = features or self.servo_features() opts = params or [] base_path = self.get_target_dir() release_path = path.join(base_path, "release", "servo") dev_path = path.join(base_path, "debug", "servo") release_exists = path.exists(release_path) dev_exists = path.exists(dev_path) if not (release or dev): if self.config["build"]["mode"] == "dev": dev = True elif self.config["build"]["mode"] == "release": release = True elif release_exists and not dev_exists: release = True elif dev_exists and not release_exists: dev = True else: print("Please specify either --dev (-d) for a development") print(" build, or --release (-r) for an optimized build.") sys.exit(1) if release and dev: print("Please specify either --dev or --release.") sys.exit(1) if target and android: print("Please specify either --target or --android.") sys.exit(1) if release: opts += ["--release"] servo_path = release_path else: servo_path = dev_path if jobs is not None: opts += ["-j", jobs] if verbose: opts += ["-v"] if android: target = self.config["android"]["target"] if target: opts += ["--target", target] self.ensure_bootstrapped(target=target) if debug_mozjs: features += ["debugmozjs"] if features: opts += ["--features", "%s" % ' '.join(features)] build_start = time() env = self.build_env(target=target, is_build=True) if with_debug_assertions: env["RUSTFLAGS"] = "-C debug_assertions" if android: # Build OpenSSL for android make_cmd = ["make"] if jobs is not None: make_cmd += ["-j" + jobs] android_dir = self.android_build_dir(dev) openssl_dir = path.join(android_dir, "native", "openssl") if not path.exists(openssl_dir): os.makedirs(openssl_dir) shutil.copy( path.join(self.android_support_dir(), "openssl.makefile"), openssl_dir) shutil.copy(path.join(self.android_support_dir(), "openssl.sh"), openssl_dir) env["ANDROID_NDK_ROOT"] = env["ANDROID_NDK"] with cd(openssl_dir): status = call(make_cmd + ["-f", "openssl.makefile"], env=env, verbose=verbose) if status: return status openssl_dir = path.join(openssl_dir, "openssl-1.0.1t") env['OPENSSL_LIB_DIR'] = openssl_dir env['OPENSSL_INCLUDE_DIR'] = path.join(openssl_dir, "include") env['OPENSSL_STATIC'] = 'TRUE' # Android builds also require having the gcc bits on the PATH and various INCLUDE # path munging if you do not want to install a standalone NDK. See: # https://dxr.mozilla.org/mozilla-central/source/build/autoconf/android.m4#139-161 os_type = platform.system().lower() if os_type not in ["linux", "darwin"]: raise Exception( "Android cross builds are only supported on Linux and macOS." ) cpu_type = platform.machine().lower() host_suffix = "unknown" if cpu_type in ["i386", "i486", "i686", "i768", "x86"]: host_suffix = "x86" elif cpu_type in ["x86_64", "x86-64", "x64", "amd64"]: host_suffix = "x86_64" host = os_type + "-" + host_suffix env['PATH'] = path.join(env['ANDROID_NDK'], "toolchains", "arm-linux-androideabi-4.9", "prebuilt", host, "bin") + ':' + env['PATH'] env['ANDROID_SYSROOT'] = path.join(env['ANDROID_NDK'], "platforms", "android-18", "arch-arm") support_include = path.join(env['ANDROID_NDK'], "sources", "android", "support", "include") cxx_include = path.join(env['ANDROID_NDK'], "sources", "cxx-stl", "llvm-libc++", "libcxx", "include") cxxabi_include = path.join(env['ANDROID_NDK'], "sources", "cxx-stl", "llvm-libc++abi", "libcxxabi", "include") env['CFLAGS'] = ' '.join( ["--sysroot", env['ANDROID_SYSROOT'], "-I" + support_include]) env['CXXFLAGS'] = ' '.join([ "--sysroot", env['ANDROID_SYSROOT'], "-I" + support_include, "-I" + cxx_include, "-I" + cxxabi_include ]) cargo_binary = "cargo" + BIN_SUFFIX if sys.platform in ("win32", "msys"): if "msvc" not in host_triple(): env[b'RUSTFLAGS'] = b'-C link-args=-Wl,--subsystem,windows' status = call([cargo_binary, "build"] + opts, env=env, cwd=self.servo_crate(), verbose=verbose) elapsed = time() - build_start # Do some additional things if the build succeeded if status == 0: if sys.platform in ("win32", "msys"): servo_exe_dir = path.join(base_path, "debug" if dev else "release") # On windows, copy in our manifest shutil.copy( path.join(self.get_top_dir(), "components", "servo", "servo.exe.manifest"), servo_exe_dir) if "msvc" in (target or host_triple()): msvc_x64 = "64" if "x86_64" in (target or host_triple()) else "" # on msvc builds, use editbin to change the subsystem to windows, but only # on release builds -- on debug builds, it hides log output if not dev: call([ "editbin", "/nologo", "/subsystem:windows", path.join(servo_exe_dir, "servo.exe") ], verbose=verbose) # on msvc, we need to copy in some DLLs in to the servo.exe dir for ssl_lib in ["ssleay32md.dll", "libeay32md.dll"]: shutil.copy( path.join(env['OPENSSL_LIB_DIR'], "../bin" + msvc_x64, ssl_lib), servo_exe_dir) elif sys.platform == "darwin": # On the Mac, set a lovely icon. This makes it easier to pick out the Servo binary in tools # like Instruments.app. try: import Cocoa icon_path = path.join(self.get_top_dir(), "resources", "servo.png") icon = Cocoa.NSImage.alloc().initWithContentsOfFile_( icon_path) if icon is not None: Cocoa.NSWorkspace.sharedWorkspace( ).setIcon_forFile_options_(icon, servo_path, 0) except ImportError: pass # Generate Desktop Notification if elapsed-time > some threshold value notify_build_done(elapsed, status == 0) print("Build %s in %s" % ("Completed" if status == 0 else "FAILED", format_duration(elapsed))) return status
def build(self, target=None, release=False, dev=False, jobs=None, features=None, android=None, magicleap=None, no_package=False, verbose=False, very_verbose=False, debug_mozjs=False, params=None, with_debug_assertions=False, libsimpleservo=False, with_frame_pointer=False): opts = params or [] if android is None: android = self.config["build"]["android"] features = features or self.servo_features() if target and android: print("Please specify either --target or --android.") sys.exit(1) # https://github.com/servo/servo/issues/22069 if debug_mozjs and magicleap: print("Please specify either --debug-mozjs or --magicleap.") sys.exit(1) if android: target = self.config["android"]["target"] if not magicleap: features += ["native-bluetooth"] if magicleap and not target: target = "aarch64-linux-android" if target and not android and not magicleap: android = self.handle_android_target(target) target_path = base_path = self.get_target_dir() if android: target_path = path.join(target_path, "android") base_path = path.join(target_path, target) elif magicleap: target_path = path.join(target_path, "magicleap") base_path = path.join(target_path, target) release_path = path.join(base_path, "release", "servo") dev_path = path.join(base_path, "debug", "servo") release_exists = path.exists(release_path) dev_exists = path.exists(dev_path) if not (release or dev): if self.config["build"]["mode"] == "dev": dev = True elif self.config["build"]["mode"] == "release": release = True elif release_exists and not dev_exists: release = True elif dev_exists and not release_exists: dev = True else: print("Please specify either --dev (-d) for a development") print(" build, or --release (-r) for an optimized build.") sys.exit(1) if release and dev: print("Please specify either --dev or --release.") sys.exit(1) if release: opts += ["--release"] servo_path = release_path else: servo_path = dev_path if jobs is not None: opts += ["-j", jobs] if verbose: opts += ["-v"] if very_verbose: opts += ["-vv"] if target: if self.config["tools"]["use-rustup"]: # 'rustup target add' fails if the toolchain is not installed at all. self.call_rustup_run(["rustc", "--version"]) check_call(["rustup" + BIN_SUFFIX, "target", "add", "--toolchain", self.toolchain(), target]) opts += ["--target", target] env = self.build_env(target=target, is_build=True) self.ensure_bootstrapped(target=target) self.ensure_clobbered() self.add_manifest_path(opts, android, libsimpleservo) if debug_mozjs: features += ["debugmozjs"] if with_frame_pointer: env['RUSTFLAGS'] = env.get('RUSTFLAGS', "") + " -C force-frame-pointers=yes" features += ["profilemozjs"] if self.config["build"]["webgl-backtrace"]: features += ["webgl-backtrace"] if self.config["build"]["dom-backtrace"]: features += ["dom-backtrace"] if features: opts += ["--features", "%s" % ' '.join(features)] build_start = time() env["CARGO_TARGET_DIR"] = target_path if with_debug_assertions: env['RUSTFLAGS'] = env.get('RUSTFLAGS', "") + " -C debug_assertions" if sys.platform == "win32": env["CC"] = "clang-cl.exe" env["CXX"] = "clang-cl.exe" if android: if "ANDROID_NDK" not in env: print("Please set the ANDROID_NDK environment variable.") sys.exit(1) if "ANDROID_SDK" not in env: print("Please set the ANDROID_SDK environment variable.") sys.exit(1) android_platform = self.config["android"]["platform"] android_toolchain_name = self.config["android"]["toolchain_name"] android_toolchain_prefix = self.config["android"]["toolchain_prefix"] android_lib = self.config["android"]["lib"] android_arch = self.config["android"]["arch"] # Build OpenSSL for android env["OPENSSL_VERSION"] = "1.0.2k" make_cmd = ["make"] if jobs is not None: make_cmd += ["-j" + jobs] openssl_dir = path.join(target_path, target, "native", "openssl") if not path.exists(openssl_dir): os.makedirs(openssl_dir) shutil.copy(path.join(self.android_support_dir(), "openssl.makefile"), openssl_dir) shutil.copy(path.join(self.android_support_dir(), "openssl.sh"), openssl_dir) # Check if the NDK version is 12 if not os.path.isfile(path.join(env["ANDROID_NDK"], 'source.properties')): print("ANDROID_NDK should have file `source.properties`.") print("The environment variable ANDROID_NDK may be set at a wrong path.") sys.exit(1) with open(path.join(env["ANDROID_NDK"], 'source.properties')) as ndk_properties: lines = ndk_properties.readlines() if lines[1].split(' = ')[1].split('.')[0] != '12': print("Currently only support NDK 12.") sys.exit(1) env["RUST_TARGET"] = target env["ANDROID_TOOLCHAIN_NAME"] = android_toolchain_name with cd(openssl_dir): status = call( make_cmd + ["-f", "openssl.makefile"], env=env, verbose=verbose) if status: return status openssl_dir = path.join(openssl_dir, "openssl-{}".format(env["OPENSSL_VERSION"])) env['OPENSSL_LIB_DIR'] = openssl_dir env['OPENSSL_INCLUDE_DIR'] = path.join(openssl_dir, "include") env['OPENSSL_STATIC'] = 'TRUE' # Android builds also require having the gcc bits on the PATH and various INCLUDE # path munging if you do not want to install a standalone NDK. See: # https://dxr.mozilla.org/mozilla-central/source/build/autoconf/android.m4#139-161 os_type = platform.system().lower() if os_type not in ["linux", "darwin"]: raise Exception("Android cross builds are only supported on Linux and macOS.") cpu_type = platform.machine().lower() host_suffix = "unknown" if cpu_type in ["i386", "i486", "i686", "i768", "x86"]: host_suffix = "x86" elif cpu_type in ["x86_64", "x86-64", "x64", "amd64"]: host_suffix = "x86_64" host = os_type + "-" + host_suffix host_cc = env.get('HOST_CC') or _get_exec_path(["clang"]) or _get_exec_path(["gcc"]) host_cxx = env.get('HOST_CXX') or _get_exec_path(["clang++"]) or _get_exec_path(["g++"]) llvm_toolchain = path.join(env['ANDROID_NDK'], "toolchains", "llvm", "prebuilt", host) gcc_toolchain = path.join(env['ANDROID_NDK'], "toolchains", android_toolchain_prefix + "-4.9", "prebuilt", host) gcc_libs = path.join(gcc_toolchain, "lib", "gcc", android_toolchain_name, "4.9.x") env['PATH'] = (path.join(llvm_toolchain, "bin") + ':' + path.join(gcc_toolchain, "bin") + ':' + env['PATH']) env['ANDROID_SYSROOT'] = path.join(env['ANDROID_NDK'], "platforms", android_platform, "arch-" + android_arch) support_include = path.join(env['ANDROID_NDK'], "sources", "android", "support", "include") cxx_include = path.join(env['ANDROID_NDK'], "sources", "cxx-stl", "llvm-libc++", "libcxx", "include") clang_include = path.join(llvm_toolchain, "lib64", "clang", "3.8", "include") sysroot_include = path.join(env['ANDROID_SYSROOT'], "usr", "include") env['HOST_CC'] = host_cc env['HOST_CXX'] = host_cxx env['HOST_CFLAGS'] = '' env['HOST_CXXFLAGS'] = '' env['CC'] = path.join(llvm_toolchain, "bin", "clang") env['CPP'] = path.join(llvm_toolchain, "bin", "clang") + " -E" env['CXX'] = path.join(llvm_toolchain, "bin", "clang++") env['ANDROID_TOOLCHAIN'] = gcc_toolchain env['GCC_TOOLCHAIN'] = gcc_toolchain gcc_toolchain_bin = path.join(gcc_toolchain, android_toolchain_name, "bin") env['AR'] = path.join(gcc_toolchain_bin, "ar") env['RANLIB'] = path.join(gcc_toolchain_bin, "ranlib") env['OBJCOPY'] = path.join(gcc_toolchain_bin, "objcopy") env['YASM'] = path.join(env['ANDROID_NDK'], 'prebuilt', host, 'bin', 'yasm') # A cheat-sheet for some of the build errors caused by getting the search path wrong... # # fatal error: 'limits' file not found # -- add -I cxx_include # unknown type name '__locale_t' (when running bindgen in mozjs_sys) # -- add -isystem sysroot_include # error: use of undeclared identifier 'UINTMAX_C' # -- add -D__STDC_CONSTANT_MACROS # # Also worth remembering: autoconf uses C for its configuration, # even for C++ builds, so the C flags need to line up with the C++ flags. env['CFLAGS'] = ' '.join([ "--target=" + target, "--sysroot=" + env['ANDROID_SYSROOT'], "--gcc-toolchain=" + gcc_toolchain, "-isystem", sysroot_include, "-L" + gcc_libs]) env['CXXFLAGS'] = ' '.join([ "--target=" + target, "--sysroot=" + env['ANDROID_SYSROOT'], "--gcc-toolchain=" + gcc_toolchain, "-I" + support_include, "-I" + cxx_include, "-I" + clang_include, "-isystem", sysroot_include, "-L" + gcc_libs, "-D__STDC_CONSTANT_MACROS", "-D__NDK_FPABI__="]) env["NDK_ANDROID_VERSION"] = android_platform.replace("android-", "") env['CPPFLAGS'] = ' '.join(["--sysroot", env['ANDROID_SYSROOT']]) env["CMAKE_ANDROID_ARCH_ABI"] = android_lib env["CMAKE_TOOLCHAIN_FILE"] = path.join(self.android_support_dir(), "toolchain.cmake") # Set output dir for gradle aar files aar_out_dir = self.android_aar_dir() if not os.path.exists(aar_out_dir): os.makedirs(aar_out_dir) env["AAR_OUT_DIR"] = aar_out_dir # GStreamer and its dependencies use pkg-config and this flag is required # to make it work in a cross-compilation context. env["PKG_CONFIG_ALLOW_CROSS"] = '1' # Build the name of the package containing all GStreamer dependencies # according to the build target. gst_lib = "gst-build-{}".format(self.config["android"]["lib"]) gst_lib_zip = "gstreamer-{}-1.14.3-20190201-081639.zip".format(self.config["android"]["lib"]) gst_dir = os.path.join(target_path, "gstreamer") gst_lib_path = os.path.join(gst_dir, gst_lib) pkg_config_path = os.path.join(gst_lib_path, "pkgconfig") env["PKG_CONFIG_PATH"] = pkg_config_path if not os.path.exists(gst_lib_path): # Download GStreamer dependencies if they have not already been downloaded # This bundle is generated with `libgstreamer_android_gen` # Follow these instructions to build and deploy new binaries # https://github.com/servo/libgstreamer_android_gen#build print("Downloading GStreamer dependencies") gst_url = "https://servo-deps.s3.amazonaws.com/gstreamer/%s" % gst_lib_zip print(gst_url) urllib.urlretrieve(gst_url, gst_lib_zip) zip_ref = zipfile.ZipFile(gst_lib_zip, "r") zip_ref.extractall(gst_dir) os.remove(gst_lib_zip) # Change pkgconfig info to make all GStreamer dependencies point # to the libgstreamer_android.so bundle. for each in os.listdir(pkg_config_path): if each.endswith('.pc'): print("Setting pkgconfig info for %s" % each) pc = os.path.join(pkg_config_path, each) expr = "s#libdir=.*#libdir=%s#g" % gst_lib_path subprocess.call(["perl", "-i", "-pe", expr, pc]) if magicleap: if platform.system() not in ["Darwin"]: raise Exception("Magic Leap builds are only supported on macOS. " "If you only wish to test if your code builds, " "run ./mach build -p libmlservo.") ml_sdk = env.get("MAGICLEAP_SDK") if not ml_sdk: raise Exception("Magic Leap builds need the MAGICLEAP_SDK environment variable") ml_support = path.join(self.get_top_dir(), "support", "magicleap") # We pretend to be an Android build env.setdefault("ANDROID_VERSION", "21") env.setdefault("ANDROID_NDK", env["MAGICLEAP_SDK"]) env.setdefault("ANDROID_NDK_VERSION", "16.0.0") env.setdefault("ANDROID_PLATFORM_DIR", path.join(env["MAGICLEAP_SDK"], "lumin")) env.setdefault("ANDROID_TOOLCHAIN_DIR", path.join(env["MAGICLEAP_SDK"], "tools", "toolchains")) env.setdefault("ANDROID_CLANG", path.join(env["ANDROID_TOOLCHAIN_DIR"], "bin", "clang")) # A random collection of search paths env.setdefault("STLPORT_LIBS", " ".join([ "-L" + path.join(env["MAGICLEAP_SDK"], "lumin", "stl", "libc++-lumin", "lib"), "-lc++" ])) env.setdefault("STLPORT_CPPFLAGS", " ".join([ "-I" + path.join(env["MAGICLEAP_SDK"], "lumin", "stl", "libc++-lumin", "include") ])) env.setdefault("CPPFLAGS", " ".join([ "--no-standard-includes", "--sysroot=" + env["ANDROID_PLATFORM_DIR"], "-I" + path.join(env["ANDROID_PLATFORM_DIR"], "usr", "include"), "-isystem" + path.join(env["ANDROID_TOOLCHAIN_DIR"], "lib64", "clang", "3.8", "include"), ])) env.setdefault("CFLAGS", " ".join([ env["CPPFLAGS"], "-L" + path.join(env["ANDROID_TOOLCHAIN_DIR"], "lib", "gcc", target, "4.9.x"), ])) env.setdefault("CXXFLAGS", " ".join([ # Sigh, Angle gets confused if there's another EGL around "-I./gfx/angle/checkout/include", env["STLPORT_CPPFLAGS"], env["CFLAGS"] ])) # The toolchain commands env.setdefault("AR", path.join(env["ANDROID_TOOLCHAIN_DIR"], "bin", "aarch64-linux-android-ar")) env.setdefault("AS", path.join(env["ANDROID_TOOLCHAIN_DIR"], "bin", "aarch64-linux-android-as")) env.setdefault("CC", path.join(env["ANDROID_TOOLCHAIN_DIR"], "bin", "aarch64-linux-android-clang")) env.setdefault("CPP", path.join(env["ANDROID_TOOLCHAIN_DIR"], "bin", "aarch64-linux-android-clang -E")) env.setdefault("CXX", path.join(env["ANDROID_TOOLCHAIN_DIR"], "bin", "aarch64-linux-android-clang++")) env.setdefault("LD", path.join(env["ANDROID_TOOLCHAIN_DIR"], "bin", "aarch64-linux-android-ld")) env.setdefault("OBJCOPY", path.join(env["ANDROID_TOOLCHAIN_DIR"], "bin", "aarch64-linux-android-objcopy")) env.setdefault("OBJDUMP", path.join(env["ANDROID_TOOLCHAIN_DIR"], "bin", "aarch64-linux-android-objdump")) env.setdefault("RANLIB", path.join(env["ANDROID_TOOLCHAIN_DIR"], "bin", "aarch64-linux-android-ranlib")) env.setdefault("STRIP", path.join(env["ANDROID_TOOLCHAIN_DIR"], "bin", "aarch64-linux-android-strip")) # Undo all of that when compiling build tools for the host env.setdefault("HOST_CFLAGS", "") env.setdefault("HOST_CXXFLAGS", "") env.setdefault("HOST_CC", "gcc") env.setdefault("HOST_CXX", "g++") env.setdefault("HOST_LD", "ld") # Some random build configurations env.setdefault("HARFBUZZ_SYS_NO_PKG_CONFIG", "1") env.setdefault("PKG_CONFIG_ALLOW_CROSS", "1") env.setdefault("CMAKE_TOOLCHAIN_FILE", path.join(ml_support, "toolchain.cmake")) # The Open SSL configuration env.setdefault("OPENSSL_DIR", path.join(target_path, target, "native", "openssl")) env.setdefault("OPENSSL_VERSION", "1.0.2k") env.setdefault("OPENSSL_STATIC", "1") # Override the linker set in .cargo/config env.setdefault("CARGO_TARGET_AARCH64_LINUX_ANDROID_LINKER", path.join(ml_support, "fake-ld.sh")) # Only build libmlservo opts += ["--package", "libmlservo"] # Download and build OpenSSL if necessary status = call(path.join(ml_support, "openssl.sh"), env=env, verbose=verbose) if status: return status if very_verbose: print (["Calling", "cargo", "build"] + opts) for key in env: print((key, env[key])) status = self.call_rustup_run(["cargo", "build"] + opts, env=env, verbose=verbose) elapsed = time() - build_start # Do some additional things if the build succeeded if status == 0: if android and not no_package: flavor = None if "googlevr" in features: flavor = "googlevr" elif "oculusvr" in features: flavor = "oculusvr" rv = Registrar.dispatch("package", context=self.context, release=release, dev=dev, target=target, flavor=flavor) if rv: return rv if sys.platform == "win32": servo_exe_dir = path.join(base_path, "debug" if dev else "release") msvc_x64 = "64" if "x86_64" in (target or host_triple()) else "" # on msvc builds, use editbin to change the subsystem to windows, but only # on release builds -- on debug builds, it hides log output if not dev: call(["editbin", "/nologo", "/subsystem:windows", path.join(servo_exe_dir, "servo.exe")], verbose=verbose) # on msvc, we need to copy in some DLLs in to the servo.exe dir for ssl_lib in ["libcryptoMD.dll", "libsslMD.dll"]: shutil.copy(path.join(env['OPENSSL_LIB_DIR'], "../bin" + msvc_x64, ssl_lib), servo_exe_dir) # Search for the generated nspr4.dll build_path = path.join(servo_exe_dir, "build") nspr4 = "nspr4.dll" nspr4_path = None for root, dirs, files in os.walk(build_path): if nspr4 in files: nspr4_path = path.join(root, nspr4) break if nspr4_path is None: print("WARNING: could not find nspr4.dll") else: shutil.copy(nspr4_path, servo_exe_dir) # copy needed gstreamer DLLs in to servo.exe dir gst_x64 = "X86_64" if msvc_x64 == "64" else "X86" gst_root = "" gst_default_path = path.join("C:\\gstreamer\\1.0", gst_x64) gst_env = "GSTREAMER_1_0_ROOT_" + gst_x64 if os.path.exists(path.join(gst_default_path, "bin", "libffi-7.dll")) or \ os.path.exists(path.join(gst_default_path, "bin", "ffi-7.dll")): gst_root = gst_default_path elif os.environ.get(gst_env) is not None: gst_root = os.environ.get(gst_env) else: print("Could not found GStreamer installation directory.") status = 1 gst_dlls = [ ["libffi-7.dll", "ffi-7.dll"], ["libgio-2.0-0.dll", "gio-2.0-0.dll"], ["libglib-2.0-0.dll", "glib-2.0-0.dll"], ["libgmodule-2.0-0.dll", "gmodule-2.0-0.dll"], ["libgobject-2.0-0.dll", "gobject-2.0-0.dll"], ["libgstapp-1.0-0.dll", "gstapp-1.0-0.dll"], ["libgstaudio-1.0-0.dll", "gstaudio-1.0-0.dll"], ["libgstbase-1.0-0.dll", "gstbase-1.0-0.dll"], ["libgstgl-1.0-0.dll", "gstgl-1.0-0.dll"], ["libgstpbutils-1.0-0.dll", "gstpbutils-1.0-0.dll"], ["libgstplayer-1.0-0.dll", "gstplayer-1.0-0.dll"], ["libgstreamer-1.0-0.dll", "gstreamer-1.0-0.dll"], ["libgstrtp-1.0-0.dll", "gstrtp-1.0-0.dll"], ["libgstsdp-1.0-0.dll", "gstsdp-1.0-0.dll"], ["libgsttag-1.0-0.dll", "gsttag-1.0-0.dll"], ["libgstvideo-1.0-0.dll", "gstvideo-1.0-0.dll"], ["libgstwebrtc-1.0-0.dll", "gstwebrtc-1.0-0.dll"], ["libintl-8.dll", "intl-8.dll"], ["liborc-0.4-0.dll", "orc-0.4-0.dll"], ["libwinpthread-1.dll", "winpthread-1.dll"], ["libz.dll", "libz-1.dll", "z-1.dll"] ] if gst_root: for gst_lib in gst_dlls: if isinstance(gst_lib, str): gst_lib = [gst_lib] for lib in gst_lib: try: shutil.copy(path.join(gst_root, "bin", lib), servo_exe_dir) break except: pass else: print("ERROR: could not find required GStreamer DLL: " + str(gst_lib)) sys.exit(1) # copy some MSVC DLLs to servo.exe dir msvc_redist_dir = None vs_platform = os.environ.get("PLATFORM", "").lower() vc_dir = os.environ.get("VCINSTALLDIR", "") vs_version = os.environ.get("VisualStudioVersion", "") msvc_deps = [ "api-ms-win-crt-runtime-l1-1-0.dll", "msvcp140.dll", "vcruntime140.dll", ] # Check if it's Visual C++ Build Tools or Visual Studio 2015 vs14_vcvars = path.join(vc_dir, "vcvarsall.bat") is_vs14 = True if os.path.isfile(vs14_vcvars) or vs_version == "14.0" else False if is_vs14: msvc_redist_dir = path.join(vc_dir, "redist", vs_platform, "Microsoft.VC140.CRT") elif vs_version == "15.0": redist_dir = path.join(os.environ.get("VCINSTALLDIR", ""), "Redist", "MSVC") if os.path.isdir(redist_dir): for p in os.listdir(redist_dir)[::-1]: redist_path = path.join(redist_dir, p) for v in ["VC141", "VC150"]: # there are two possible paths # `x64\Microsoft.VC*.CRT` or `onecore\x64\Microsoft.VC*.CRT` redist1 = path.join(redist_path, vs_platform, "Microsoft.{}.CRT".format(v)) redist2 = path.join(redist_path, "onecore", vs_platform, "Microsoft.{}.CRT".format(v)) if os.path.isdir(redist1): msvc_redist_dir = redist1 break elif os.path.isdir(redist2): msvc_redist_dir = redist2 break if msvc_redist_dir: break if msvc_redist_dir: redist_dirs = [ msvc_redist_dir, path.join(os.environ["WindowsSdkDir"], "Redist", "ucrt", "DLLs", vs_platform), ] for msvc_dll in msvc_deps: dll_found = False for dll_dir in redist_dirs: dll = path.join(dll_dir, msvc_dll) servo_dir_dll = path.join(servo_exe_dir, msvc_dll) if os.path.isfile(dll): if os.path.isfile(servo_dir_dll): # avoid permission denied error when overwrite dll in servo build directory os.chmod(servo_dir_dll, stat.S_IWUSR) shutil.copy(dll, servo_exe_dir) dll_found = True break if not dll_found: print("DLL file `{}` not found!".format(msvc_dll)) status = 1 elif sys.platform == "darwin": # On the Mac, set a lovely icon. This makes it easier to pick out the Servo binary in tools # like Instruments.app. try: import Cocoa icon_path = path.join(self.get_top_dir(), "resources", "servo.png") icon = Cocoa.NSImage.alloc().initWithContentsOfFile_(icon_path) if icon is not None: Cocoa.NSWorkspace.sharedWorkspace().setIcon_forFile_options_(icon, servo_path, 0) except ImportError: pass # Generate Desktop Notification if elapsed-time > some threshold value notify_build_done(self.config, elapsed, status == 0) print("Build %s in %s" % ("Completed" if status == 0 else "FAILED", format_duration(elapsed))) return status