def get_rstcheck_binary():
"""
Returns the path of the first rstcheck binary available
if not found returns None
"""
binary = os.environ.get('RSTCHECK')
if binary:
return binary
return which('rstcheck')
def get_shellcheck_binary():
"""
Returns the path of the first shellcheck binary available
if not found returns None
"""
binary = os.environ.get('SHELLCHECK')
if binary:
return binary
return which('shellcheck')
def get_rustfmt_binary():
"""
Returns the path of the first rustfmt binary available
if not found returns None
"""
binary = os.environ.get("RUSTFMT")
if binary:
return binary
return which("rustfmt")
def get_cargo_path(self):
try:
return self.substs['CARGO']
except (BuildEnvironmentNotFoundException, KeyError):
# Default if this tree isn't configured.
from mozfile import which
cargo = which('cargo')
if not cargo:
raise OSError(errno.ENOENT, "Could not find 'cargo' on your $PATH.")
return cargo
def get_codespell_binary():
"""
Returns the path of the first codespell binary available
if not found returns None
"""
binary = os.environ.get("CODESPELL")
if binary:
return binary
return which("codespell")
def is_nasm_modern(self):
nasm = which("nasm")
if not nasm:
return False
our = self._parse_version_short(nasm, "version")
if not our:
return False
return our >= MODERN_NASM_VERSION
def run_as_root(self, command):
if os.geteuid() != 0:
if which("sudo"):
command.insert(0, "sudo")
else:
command = ["su", "root", "-c", " ".join(command)]
print("Executing as root:", subprocess.list2cmdline(command))
subprocess.check_call(command, stdin=sys.stdin)
def find_sdk_tool(binary, log=None):
if binary.lower().endswith(".exe"):
binary = binary[:-4]
maybe = os.environ.get(binary.upper())
if maybe:
log(
logging.DEBUG,
"msix",
{"binary": binary, "path": maybe},
"Found {binary} in environment: {path}",
)
return mozpath.normsep(maybe)
maybe = which(
binary, extra_search_dirs=["c:/Windows/System32/WindowsPowershell/v1.0"]
)
if maybe:
log(
logging.DEBUG,
"msix",
{"binary": binary, "path": maybe},
"Found {binary} on path: {path}",
)
return mozpath.normsep(maybe)
sdk = os.environ.get("WINDOWSSDKDIR") or "C:/Program Files (x86)/Windows Kits/10"
log(
logging.DEBUG,
"msix",
{"binary": binary, "sdk": sdk},
"Looking for {binary} in Windows SDK: {sdk}",
)
if sdk:
# Like `bin/VERSION/ARCH/tool.exe`.
finder = FileFinder(sdk)
# TODO: handle running on ARM.
is_64bits = sys.maxsize > 2 ** 32
arch = "x64" if is_64bits else "x86"
for p, f in finder.find(
"bin/**/{arch}/{binary}.exe".format(arch=arch, binary=binary)
):
maybe = mozpath.normsep(mozpath.join(sdk, p))
log(
logging.DEBUG,
"msix",
{"binary": binary, "path": maybe},
"Found {binary} in Windows SDK: {path}",
)
return maybe
return None
def find_python3_executable(min_version='3.5.0'):
"""Find a Python 3 executable.
Returns a tuple containing the the path to an executable binary and a
version tuple. Both tuple entries will be None if a Python executable
could not be resolved.
"""
from mozfile import which
if not min_version.startswith('3.'):
raise ValueError('min_version expected a 3.x string, got %s' %
min_version)
min_version = StrictVersion(min_version)
if sys.version_info.major >= 3:
our_version = StrictVersion('%s.%s.%s' % (sys.version_info[0:3]))
if our_version >= min_version:
# This will potentially return a virtualenv Python. It's probably
# OK for now...
return sys.executable, our_version.version
# Else fall back to finding another binary.
# https://www.python.org/dev/peps/pep-0394/ defines how the Python binary
# should be named. `python3` should refer to some Python 3. `python` may
# refer to a Python 2 or 3. `pythonX.Y` may exist.
#
# Since `python` is ambiguous and `python3` should always exist, we
# ignore `python` here. We instead look for the preferred `python3` first
# and fall back to `pythonX.Y` if it isn't found or doesn't meet our
# version requirements.
names = ['python3']
# Look for `python3.Y` down to our minimum version.
for minor in range(9, min_version.version[1] - 1, -1):
names.append('python3.%d' % minor)
for name in names:
exe = which(name)
if not exe:
continue
# We always verify we can invoke the executable and its version is
# sane.
try:
version = python_executable_version(exe)
except (subprocess.CalledProcessError, ValueError):
continue
if version >= min_version:
return exe, version.version
return None, None
def dnf_groupinstall(self, *packages):
if which("dnf"):
command = ["dnf", "groupinstall"]
else:
command = ["yum", "groupinstall"]
if self.no_interactive:
command.append("-y")
command.extend(packages)
self.run_as_root(command)
def get_rustfmt_binary():
"""
Returns the path of the first rustfmt binary available
if not found returns None
"""
binary = os.environ.get("RUSTFMT")
if binary:
return binary
rust_path = os.path.join(get_tools_dir(), "rustc", "bin")
return which("rustfmt", path=os.pathsep.join([rust_path, os.environ["PATH"]]))
def is_rust_modern(self, cargo_bin):
rustc = which('rustc', extra_search_dirs=[cargo_bin])
if not rustc:
print('Could not find a Rust compiler.')
return False, None
our = self._parse_version(rustc)
if not our:
return False, None
return our >= MODERN_RUST_VERSION, our
def dnf_update(self, *packages):
if which('dnf'):
command = ['dnf', 'update']
else:
command = ['yum', 'update']
if self.no_interactive:
command.append('-y')
command.extend(packages)
self.run_as_root(command)
def is_mercurial_modern(self):
hg = which('hg')
if not hg:
print(NO_MERCURIAL)
return False, False, None
our = self._parse_version(hg, 'version', self._hg_cleanenv())
if not our:
return True, False, None
return True, our >= MODERN_MERCURIAL_VERSION, our
def get_tool_path(tool):
"""Obtain the path of `tool`."""
if os.path.isabs(tool) and os.path.exists(tool):
return tool
path = which(tool)
if not path:
raise MissingVCSTool('Unable to obtain %s path. Try running '
'|mach bootstrap| to ensure your environment is up to '
'date.' % tool)
return path
def dnf_groupinstall(self, *packages):
if which('dnf'):
command = ['dnf', 'groupinstall']
else:
command = ['yum', 'groupinstall']
if self.no_interactive:
command.append('-y')
command.extend(packages)
self.run_as_root(command)
def setup(self):
self.metrics = {}
self.metrics_fields = []
# making sure we have ffmpeg and imagemagick available
for tool in ("ffmpeg", "convert"):
if sys.platform in ("win32", "msys"):
tool += ".exe"
path = which(tool)
if not path:
raise OSError(errno.ENOENT, f"Could not find {tool}")
def dnf_update(self, *packages):
if which("dnf"):
command = ["dnf", "update"]
else:
command = ["yum", "update"]
if self.no_interactive:
command.append("-y")
command.extend(packages)
self.run_as_root(command)
def check_code_submission(self, checkout_root):
if self.instance.no_interactive or which("moz-phab"):
return
if not self.instance.prompt_yesno(
"Will you be submitting commits to Mozilla?"):
return
mach_binary = os.path.join(checkout_root, "mach")
subprocess.check_call(
(sys.executable, mach_binary, "install-moz-phab"))
def _ensure_macports_packages(self, packages):
self.port = which("port")
assert self.port is not None
installed = set(
subprocess.check_output([self.port, "installed"],
universal_newlines=True).split())
missing = [package for package in packages if package not in installed]
if missing:
print(PACKAGE_MANAGER_PACKAGES % ("MacPorts", ))
self.run_as_root([self.port, "-v", "install"] + missing)
def python(self, no_virtualenv, exec_file, ipython, args):
# Avoid logging the command
self.log_manager.terminal_handler.setLevel(logging.CRITICAL)
# Note: subprocess requires native strings in os.environ on Windows.
append_env = {
'PYTHONDONTWRITEBYTECODE': str('1'),
}
if no_virtualenv:
python_path = sys.executable
append_env['PYTHONPATH'] = os.pathsep.join(sys.path)
else:
self._activate_virtualenv()
python_path = self.virtualenv_manager.python_path
if exec_file:
exec(open(exec_file).read())
return 0
if ipython:
bindir = os.path.dirname(python_path)
python_path = which('ipython', path=bindir)
if not python_path:
if not no_virtualenv:
# Use `_run_pip` directly rather than `install_pip_package` to bypass
# `req.check_if_exists()` which may detect a system installed ipython.
self.virtualenv_manager._run_pip(['install', 'ipython'])
python_path = which('ipython', path=bindir)
if not python_path:
print("error: could not detect or install ipython")
return 1
return self.run_process(
[python_path] + args,
pass_thru=True, # Allow user to run Python interactively.
ensure_exit_code=False, # Don't throw on non-zero exit code.
python_unbuffered=False, # Leave input buffered.
append_env=append_env)
def ret(min_version=min_versions[major]):
from mozfile import which
prefix = min_version[0] + '.'
if not min_version.startswith(prefix):
raise ValueError('min_version expected a %sx string, got %s' %
(prefix, min_version))
min_version = StrictVersion(min_version)
major = min_version.version[0]
if sys.version_info.major == major:
our_version = StrictVersion('%s.%s.%s' % (sys.version_info[0:3]))
if our_version >= min_version:
# This will potentially return a virtualenv Python. It's probably
# OK for now...
return sys.executable, our_version.version
# Else fall back to finding another binary.
# https://www.python.org/dev/peps/pep-0394/ defines how the Python binary
# should be named. `python3` should refer to some Python 3, and
# `python2` to some Python 2. `python` may refer to a Python 2 or 3.
# `pythonX.Y` may exist.
#
# Since `python` is ambiguous and `pythonX` should always exist, we
# ignore `python` here. We instead look for the preferred `pythonX` first
# and fall back to `pythonX.Y` if it isn't found or doesn't meet our
# version requirements.
names = ['python%d' % major]
# Look for `pythonX.Y` down to our minimum version.
for minor in range(9, min_version.version[1] - 1, -1):
names.append('python%d.%d' % (major, minor))
for name in names:
exe = which(name)
if not exe:
continue
# We always verify we can invoke the executable and its version is
# sane.
try:
version = python_executable_version(exe)
except (subprocess.CalledProcessError, ValueError):
continue
if version >= min_version:
return exe, version.version
return None, None
def check_jsdoc(self):
try:
from mozfile import which
exe_name = which('jsdoc')
if not exe_name:
return 1
out = subprocess.check_output([exe_name, '--version'])
version = out.split()[1]
except subprocess.CalledProcessError:
version = None
if not version or not version.startswith(b'3.5'):
return 1
def _make_path(self):
baseconfig = os.path.join(self.topsrcdir, 'config', 'baseconfig.mk')
def is_xcode_lisense_error(output):
return self._is_osx() and b'Agreeing to the Xcode' in output
def validate_make(make):
if os.path.exists(baseconfig) and os.path.exists(make):
cmd = [make, '-f', baseconfig]
if self._is_windows():
cmd.append('HOST_OS_ARCH=WINNT')
try:
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
return False, is_xcode_lisense_error(e.output)
return True, False
return False, False
xcode_lisense_error = False
possible_makes = [
'gmake', 'make', 'mozmake', 'gnumake', 'mingw32-make'
]
if 'MAKE' in os.environ:
make = os.environ['MAKE']
possible_makes.insert(0, make)
for test in possible_makes:
if os.path.isabs(test):
make = test
else:
make = which(test)
if not make:
continue
result, xcode_lisense_error_tmp = validate_make(make)
if result:
return [make]
if xcode_lisense_error_tmp:
xcode_lisense_error = True
if xcode_lisense_error:
raise Exception(
'Xcode requires accepting to the license agreement.\n'
'Please run Xcode and accept the license agreement.')
if self._is_windows():
raise Exception('Could not find a suitable make implementation.\n'
'Please use MozillaBuild 1.9 or newer')
else:
raise Exception('Could not find a suitable make implementation.')
def get_cargo_binary(log):
"""
Returns the path of the first rustfmt binary available
if not found returns None
"""
cargo_home = os.environ.get("CARGO_HOME")
if cargo_home:
log.debug("Found CARGO_HOME in {}".format(cargo_home))
cargo_bin = os.path.join(cargo_home, "bin", "cargo")
if os.path.exists(cargo_bin):
return cargo_bin
log.debug("Did not find {} in CARGO_HOME".format(cargo_bin))
return None
return which("cargo")
def lint(paths, config, fix=None, **lintargs):
log = lintargs["log"]
binary = get_codespell_binary()
if not binary:
print(CODESPELL_NOT_FOUND)
if "MOZ_AUTOMATION" in os.environ:
return 1
return []
config["root"] = lintargs["root"]
exclude_list = os.path.join(here, "exclude-list.txt")
cmd_args = [
which("python"),
binary,
"--disable-colors",
# Silence some warnings:
# 1: disable warnings about wrong encoding
# 2: disable warnings about binary file
# 4: shut down warnings about automatic fixes
# that were disabled in dictionary.
"--quiet-level=7",
"--ignore-words=" + exclude_list,
]
if "exclude" in config:
cmd_args.append("--skip=*.dic,{}".format(",".join(config["exclude"])))
log.debug("Command: {}".format(" ".join(cmd_args)))
log.debug("Version: {}".format(get_codespell_version(binary)))
if fix:
CodespellProcess._fix = True
base_command = cmd_args + paths
run_process(config, base_command)
if fix:
global results
results = []
cmd_args.append("--write-changes")
log.debug("Command: {}".format(" ".join(cmd_args)))
log.debug("Version: {}".format(get_codespell_version(binary)))
base_command = cmd_args + paths
run_process(config, base_command)
CodespellProcess.fixed = CodespellProcess.fixed - len(results)
else:
CodespellProcess.fixed = 0
return {"results": results, "fixed": CodespellProcess.fixed}
def __init__(self, **kwargs):
if 'MOZ_WINDOWS_BOOTSTRAP' not in os.environ or os.environ[
'MOZ_WINDOWS_BOOTSTRAP'] != '1':
raise NotImplementedError(
'Bootstrap support for Windows is under development. For '
'now use MozillaBuild to set up a build environment on '
'Windows. If you are testing Windows Bootstrap support, '
'try `export MOZ_WINDOWS_BOOTSTRAP=1`')
BaseBootstrapper.__init__(self, **kwargs)
if not which('pacman'):
raise NotImplementedError(
'The Windows bootstrapper only works with msys2 with '
'pacman. Get msys2 at http://msys2.github.io/')
print('Using an experimental bootstrapper for Windows.')
def check_code_submission(self, checkout_root):
if self.instance.no_interactive or which("moz-phab"):
return
# Skip moz-phab install until bug 1696357 is fixed and makes it to a moz-phab
# release.
if sys.platform.startswith("darwin") and platform.machine() == "arm64":
return
if not self.instance.prompt_yesno("Will you be submitting commits to Mozilla?"):
return
mach_binary = os.path.join(checkout_root, "mach")
subprocess.check_call((sys.executable, mach_binary, "install-moz-phab"))
def _make_path(self):
baseconfig = os.path.join(self.topsrcdir, "config", "baseconfig.mk")
def is_xcode_lisense_error(output):
return self._is_osx() and b"Agreeing to the Xcode" in output
def validate_make(make):
if os.path.exists(baseconfig) and os.path.exists(make):
cmd = [make, "-f", baseconfig]
if self._is_windows():
cmd.append("HOST_OS_ARCH=WINNT")
try:
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
return False, is_xcode_lisense_error(e.output)
return True, False
return False, False
xcode_lisense_error = False
possible_makes = [
"gmake", "make", "mozmake", "gnumake", "mingw32-make"
]
if "MAKE" in os.environ:
make = os.environ["MAKE"]
possible_makes.insert(0, make)
for test in possible_makes:
if os.path.isabs(test):
make = test
else:
make = which(test)
if not make:
continue
result, xcode_lisense_error_tmp = validate_make(make)
if result:
return make
if xcode_lisense_error_tmp:
xcode_lisense_error = True
if xcode_lisense_error:
raise Exception(
"Xcode requires accepting to the license agreement.\n"
"Please run Xcode and accept the license agreement.")
if self._is_windows():
raise Exception("Could not find a suitable make implementation.\n"
"Please use MozillaBuild 1.9 or newer")
else:
raise Exception("Could not find a suitable make implementation.")
def vcs_setup(self, update_only=False):
"""Ensure a Version Control System (Mercurial or Git) is optimally
configured.
This command will inspect your VCS configuration and
guide you through an interactive wizard helping you configure the
VCS for optimal use on Mozilla projects.
User choice is respected: no changes are made without explicit
confirmation from you.
If "--update-only" is used, the interactive wizard is disabled
and this command only ensures that remote repositories providing
VCS extensions are up to date.
"""
import mozboot.bootstrap as bootstrap
import mozversioncontrol
from mozfile import which
repo = mozversioncontrol.get_repository_object(
self._mach_context.topdir)
tool = 'hg'
if repo.name == 'git':
tool = 'git'
# "hg" is an executable script with a shebang, which will be found by
# which. We need to pass a win32 executable to the function because we
# spawn a process from it.
if sys.platform in ('win32', 'msys'):
tool += '.exe'
vcs = which(tool)
if not vcs:
raise OSError(errno.ENOENT,
"Could not find {} on $PATH".format(tool))
if update_only:
if repo.name == 'git':
bootstrap.update_git_tools(vcs, self._mach_context.state_dir,
self._mach_context.topdir)
else:
bootstrap.update_vct(vcs, self._mach_context.state_dir)
else:
if repo.name == 'git':
bootstrap.configure_git(vcs, self._mach_context.state_dir,
self._mach_context.topdir)
else:
bootstrap.configure_mercurial(vcs,
self._mach_context.state_dir)
