def get_rstcheck_binary(): """ Returns the path of the first rstcheck binary available if not found returns None """ binary = os.environ.get('RSTCHECK') if binary: return binary return which('rstcheck')
def get_shellcheck_binary(): """ Returns the path of the first shellcheck binary available if not found returns None """ binary = os.environ.get('SHELLCHECK') if binary: return binary return which('shellcheck')
def get_rustfmt_binary(): """ Returns the path of the first rustfmt binary available if not found returns None """ binary = os.environ.get("RUSTFMT") if binary: return binary return which("rustfmt")
def get_cargo_path(self): try: return self.substs['CARGO'] except (BuildEnvironmentNotFoundException, KeyError): # Default if this tree isn't configured. from mozfile import which cargo = which('cargo') if not cargo: raise OSError(errno.ENOENT, "Could not find 'cargo' on your $PATH.") return cargo
def get_codespell_binary(): """ Returns the path of the first codespell binary available if not found returns None """ binary = os.environ.get("CODESPELL") if binary: return binary return which("codespell")
def is_nasm_modern(self): nasm = which("nasm") if not nasm: return False our = self._parse_version_short(nasm, "version") if not our: return False return our >= MODERN_NASM_VERSION
def run_as_root(self, command): if os.geteuid() != 0: if which("sudo"): command.insert(0, "sudo") else: command = ["su", "root", "-c", " ".join(command)] print("Executing as root:", subprocess.list2cmdline(command)) subprocess.check_call(command, stdin=sys.stdin)
def find_sdk_tool(binary, log=None): if binary.lower().endswith(".exe"): binary = binary[:-4] maybe = os.environ.get(binary.upper()) if maybe: log( logging.DEBUG, "msix", {"binary": binary, "path": maybe}, "Found {binary} in environment: {path}", ) return mozpath.normsep(maybe) maybe = which( binary, extra_search_dirs=["c:/Windows/System32/WindowsPowershell/v1.0"] ) if maybe: log( logging.DEBUG, "msix", {"binary": binary, "path": maybe}, "Found {binary} on path: {path}", ) return mozpath.normsep(maybe) sdk = os.environ.get("WINDOWSSDKDIR") or "C:/Program Files (x86)/Windows Kits/10" log( logging.DEBUG, "msix", {"binary": binary, "sdk": sdk}, "Looking for {binary} in Windows SDK: {sdk}", ) if sdk: # Like `bin/VERSION/ARCH/tool.exe`. finder = FileFinder(sdk) # TODO: handle running on ARM. is_64bits = sys.maxsize > 2 ** 32 arch = "x64" if is_64bits else "x86" for p, f in finder.find( "bin/**/{arch}/{binary}.exe".format(arch=arch, binary=binary) ): maybe = mozpath.normsep(mozpath.join(sdk, p)) log( logging.DEBUG, "msix", {"binary": binary, "path": maybe}, "Found {binary} in Windows SDK: {path}", ) return maybe return None
def find_python3_executable(min_version='3.5.0'): """Find a Python 3 executable. Returns a tuple containing the the path to an executable binary and a version tuple. Both tuple entries will be None if a Python executable could not be resolved. """ from mozfile import which if not min_version.startswith('3.'): raise ValueError('min_version expected a 3.x string, got %s' % min_version) min_version = StrictVersion(min_version) if sys.version_info.major >= 3: our_version = StrictVersion('%s.%s.%s' % (sys.version_info[0:3])) if our_version >= min_version: # This will potentially return a virtualenv Python. It's probably # OK for now... return sys.executable, our_version.version # Else fall back to finding another binary. # https://www.python.org/dev/peps/pep-0394/ defines how the Python binary # should be named. `python3` should refer to some Python 3. `python` may # refer to a Python 2 or 3. `pythonX.Y` may exist. # # Since `python` is ambiguous and `python3` should always exist, we # ignore `python` here. We instead look for the preferred `python3` first # and fall back to `pythonX.Y` if it isn't found or doesn't meet our # version requirements. names = ['python3'] # Look for `python3.Y` down to our minimum version. for minor in range(9, min_version.version[1] - 1, -1): names.append('python3.%d' % minor) for name in names: exe = which(name) if not exe: continue # We always verify we can invoke the executable and its version is # sane. try: version = python_executable_version(exe) except (subprocess.CalledProcessError, ValueError): continue if version >= min_version: return exe, version.version return None, None
def dnf_groupinstall(self, *packages): if which("dnf"): command = ["dnf", "groupinstall"] else: command = ["yum", "groupinstall"] if self.no_interactive: command.append("-y") command.extend(packages) self.run_as_root(command)
def get_rustfmt_binary(): """ Returns the path of the first rustfmt binary available if not found returns None """ binary = os.environ.get("RUSTFMT") if binary: return binary rust_path = os.path.join(get_tools_dir(), "rustc", "bin") return which("rustfmt", path=os.pathsep.join([rust_path, os.environ["PATH"]]))
def is_rust_modern(self, cargo_bin): rustc = which('rustc', extra_search_dirs=[cargo_bin]) if not rustc: print('Could not find a Rust compiler.') return False, None our = self._parse_version(rustc) if not our: return False, None return our >= MODERN_RUST_VERSION, our
def dnf_update(self, *packages): if which('dnf'): command = ['dnf', 'update'] else: command = ['yum', 'update'] if self.no_interactive: command.append('-y') command.extend(packages) self.run_as_root(command)
def is_mercurial_modern(self): hg = which('hg') if not hg: print(NO_MERCURIAL) return False, False, None our = self._parse_version(hg, 'version', self._hg_cleanenv()) if not our: return True, False, None return True, our >= MODERN_MERCURIAL_VERSION, our
def get_tool_path(tool): """Obtain the path of `tool`.""" if os.path.isabs(tool) and os.path.exists(tool): return tool path = which(tool) if not path: raise MissingVCSTool('Unable to obtain %s path. Try running ' '|mach bootstrap| to ensure your environment is up to ' 'date.' % tool) return path
def dnf_groupinstall(self, *packages): if which('dnf'): command = ['dnf', 'groupinstall'] else: command = ['yum', 'groupinstall'] if self.no_interactive: command.append('-y') command.extend(packages) self.run_as_root(command)
def setup(self): self.metrics = {} self.metrics_fields = [] # making sure we have ffmpeg and imagemagick available for tool in ("ffmpeg", "convert"): if sys.platform in ("win32", "msys"): tool += ".exe" path = which(tool) if not path: raise OSError(errno.ENOENT, f"Could not find {tool}")
def dnf_update(self, *packages): if which("dnf"): command = ["dnf", "update"] else: command = ["yum", "update"] if self.no_interactive: command.append("-y") command.extend(packages) self.run_as_root(command)
def check_code_submission(self, checkout_root): if self.instance.no_interactive or which("moz-phab"): return if not self.instance.prompt_yesno( "Will you be submitting commits to Mozilla?"): return mach_binary = os.path.join(checkout_root, "mach") subprocess.check_call( (sys.executable, mach_binary, "install-moz-phab"))
def _ensure_macports_packages(self, packages): self.port = which("port") assert self.port is not None installed = set( subprocess.check_output([self.port, "installed"], universal_newlines=True).split()) missing = [package for package in packages if package not in installed] if missing: print(PACKAGE_MANAGER_PACKAGES % ("MacPorts", )) self.run_as_root([self.port, "-v", "install"] + missing)
def python(self, no_virtualenv, exec_file, ipython, args): # Avoid logging the command self.log_manager.terminal_handler.setLevel(logging.CRITICAL) # Note: subprocess requires native strings in os.environ on Windows. append_env = { 'PYTHONDONTWRITEBYTECODE': str('1'), } if no_virtualenv: python_path = sys.executable append_env['PYTHONPATH'] = os.pathsep.join(sys.path) else: self._activate_virtualenv() python_path = self.virtualenv_manager.python_path if exec_file: exec(open(exec_file).read()) return 0 if ipython: bindir = os.path.dirname(python_path) python_path = which('ipython', path=bindir) if not python_path: if not no_virtualenv: # Use `_run_pip` directly rather than `install_pip_package` to bypass # `req.check_if_exists()` which may detect a system installed ipython. self.virtualenv_manager._run_pip(['install', 'ipython']) python_path = which('ipython', path=bindir) if not python_path: print("error: could not detect or install ipython") return 1 return self.run_process( [python_path] + args, pass_thru=True, # Allow user to run Python interactively. ensure_exit_code=False, # Don't throw on non-zero exit code. python_unbuffered=False, # Leave input buffered. append_env=append_env)
def ret(min_version=min_versions[major]): from mozfile import which prefix = min_version[0] + '.' if not min_version.startswith(prefix): raise ValueError('min_version expected a %sx string, got %s' % (prefix, min_version)) min_version = StrictVersion(min_version) major = min_version.version[0] if sys.version_info.major == major: our_version = StrictVersion('%s.%s.%s' % (sys.version_info[0:3])) if our_version >= min_version: # This will potentially return a virtualenv Python. It's probably # OK for now... return sys.executable, our_version.version # Else fall back to finding another binary. # https://www.python.org/dev/peps/pep-0394/ defines how the Python binary # should be named. `python3` should refer to some Python 3, and # `python2` to some Python 2. `python` may refer to a Python 2 or 3. # `pythonX.Y` may exist. # # Since `python` is ambiguous and `pythonX` should always exist, we # ignore `python` here. We instead look for the preferred `pythonX` first # and fall back to `pythonX.Y` if it isn't found or doesn't meet our # version requirements. names = ['python%d' % major] # Look for `pythonX.Y` down to our minimum version. for minor in range(9, min_version.version[1] - 1, -1): names.append('python%d.%d' % (major, minor)) for name in names: exe = which(name) if not exe: continue # We always verify we can invoke the executable and its version is # sane. try: version = python_executable_version(exe) except (subprocess.CalledProcessError, ValueError): continue if version >= min_version: return exe, version.version return None, None
def check_jsdoc(self): try: from mozfile import which exe_name = which('jsdoc') if not exe_name: return 1 out = subprocess.check_output([exe_name, '--version']) version = out.split()[1] except subprocess.CalledProcessError: version = None if not version or not version.startswith(b'3.5'): return 1
def _make_path(self): baseconfig = os.path.join(self.topsrcdir, 'config', 'baseconfig.mk') def is_xcode_lisense_error(output): return self._is_osx() and b'Agreeing to the Xcode' in output def validate_make(make): if os.path.exists(baseconfig) and os.path.exists(make): cmd = [make, '-f', baseconfig] if self._is_windows(): cmd.append('HOST_OS_ARCH=WINNT') try: subprocess.check_output(cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: return False, is_xcode_lisense_error(e.output) return True, False return False, False xcode_lisense_error = False possible_makes = [ 'gmake', 'make', 'mozmake', 'gnumake', 'mingw32-make' ] if 'MAKE' in os.environ: make = os.environ['MAKE'] possible_makes.insert(0, make) for test in possible_makes: if os.path.isabs(test): make = test else: make = which(test) if not make: continue result, xcode_lisense_error_tmp = validate_make(make) if result: return [make] if xcode_lisense_error_tmp: xcode_lisense_error = True if xcode_lisense_error: raise Exception( 'Xcode requires accepting to the license agreement.\n' 'Please run Xcode and accept the license agreement.') if self._is_windows(): raise Exception('Could not find a suitable make implementation.\n' 'Please use MozillaBuild 1.9 or newer') else: raise Exception('Could not find a suitable make implementation.')
def get_cargo_binary(log): """ Returns the path of the first rustfmt binary available if not found returns None """ cargo_home = os.environ.get("CARGO_HOME") if cargo_home: log.debug("Found CARGO_HOME in {}".format(cargo_home)) cargo_bin = os.path.join(cargo_home, "bin", "cargo") if os.path.exists(cargo_bin): return cargo_bin log.debug("Did not find {} in CARGO_HOME".format(cargo_bin)) return None return which("cargo")
def lint(paths, config, fix=None, **lintargs): log = lintargs["log"] binary = get_codespell_binary() if not binary: print(CODESPELL_NOT_FOUND) if "MOZ_AUTOMATION" in os.environ: return 1 return [] config["root"] = lintargs["root"] exclude_list = os.path.join(here, "exclude-list.txt") cmd_args = [ which("python"), binary, "--disable-colors", # Silence some warnings: # 1: disable warnings about wrong encoding # 2: disable warnings about binary file # 4: shut down warnings about automatic fixes # that were disabled in dictionary. "--quiet-level=7", "--ignore-words=" + exclude_list, ] if "exclude" in config: cmd_args.append("--skip=*.dic,{}".format(",".join(config["exclude"]))) log.debug("Command: {}".format(" ".join(cmd_args))) log.debug("Version: {}".format(get_codespell_version(binary))) if fix: CodespellProcess._fix = True base_command = cmd_args + paths run_process(config, base_command) if fix: global results results = [] cmd_args.append("--write-changes") log.debug("Command: {}".format(" ".join(cmd_args))) log.debug("Version: {}".format(get_codespell_version(binary))) base_command = cmd_args + paths run_process(config, base_command) CodespellProcess.fixed = CodespellProcess.fixed - len(results) else: CodespellProcess.fixed = 0 return {"results": results, "fixed": CodespellProcess.fixed}
def __init__(self, **kwargs): if 'MOZ_WINDOWS_BOOTSTRAP' not in os.environ or os.environ[ 'MOZ_WINDOWS_BOOTSTRAP'] != '1': raise NotImplementedError( 'Bootstrap support for Windows is under development. For ' 'now use MozillaBuild to set up a build environment on ' 'Windows. If you are testing Windows Bootstrap support, ' 'try `export MOZ_WINDOWS_BOOTSTRAP=1`') BaseBootstrapper.__init__(self, **kwargs) if not which('pacman'): raise NotImplementedError( 'The Windows bootstrapper only works with msys2 with ' 'pacman. Get msys2 at http://msys2.github.io/') print('Using an experimental bootstrapper for Windows.')
def check_code_submission(self, checkout_root): if self.instance.no_interactive or which("moz-phab"): return # Skip moz-phab install until bug 1696357 is fixed and makes it to a moz-phab # release. if sys.platform.startswith("darwin") and platform.machine() == "arm64": return if not self.instance.prompt_yesno("Will you be submitting commits to Mozilla?"): return mach_binary = os.path.join(checkout_root, "mach") subprocess.check_call((sys.executable, mach_binary, "install-moz-phab"))
def _make_path(self): baseconfig = os.path.join(self.topsrcdir, "config", "baseconfig.mk") def is_xcode_lisense_error(output): return self._is_osx() and b"Agreeing to the Xcode" in output def validate_make(make): if os.path.exists(baseconfig) and os.path.exists(make): cmd = [make, "-f", baseconfig] if self._is_windows(): cmd.append("HOST_OS_ARCH=WINNT") try: subprocess.check_output(cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: return False, is_xcode_lisense_error(e.output) return True, False return False, False xcode_lisense_error = False possible_makes = [ "gmake", "make", "mozmake", "gnumake", "mingw32-make" ] if "MAKE" in os.environ: make = os.environ["MAKE"] possible_makes.insert(0, make) for test in possible_makes: if os.path.isabs(test): make = test else: make = which(test) if not make: continue result, xcode_lisense_error_tmp = validate_make(make) if result: return make if xcode_lisense_error_tmp: xcode_lisense_error = True if xcode_lisense_error: raise Exception( "Xcode requires accepting to the license agreement.\n" "Please run Xcode and accept the license agreement.") if self._is_windows(): raise Exception("Could not find a suitable make implementation.\n" "Please use MozillaBuild 1.9 or newer") else: raise Exception("Could not find a suitable make implementation.")
def vcs_setup(self, update_only=False): """Ensure a Version Control System (Mercurial or Git) is optimally configured. This command will inspect your VCS configuration and guide you through an interactive wizard helping you configure the VCS for optimal use on Mozilla projects. User choice is respected: no changes are made without explicit confirmation from you. If "--update-only" is used, the interactive wizard is disabled and this command only ensures that remote repositories providing VCS extensions are up to date. """ import mozboot.bootstrap as bootstrap import mozversioncontrol from mozfile import which repo = mozversioncontrol.get_repository_object( self._mach_context.topdir) tool = 'hg' if repo.name == 'git': tool = 'git' # "hg" is an executable script with a shebang, which will be found by # which. We need to pass a win32 executable to the function because we # spawn a process from it. if sys.platform in ('win32', 'msys'): tool += '.exe' vcs = which(tool) if not vcs: raise OSError(errno.ENOENT, "Could not find {} on $PATH".format(tool)) if update_only: if repo.name == 'git': bootstrap.update_git_tools(vcs, self._mach_context.state_dir, self._mach_context.topdir) else: bootstrap.update_vct(vcs, self._mach_context.state_dir) else: if repo.name == 'git': bootstrap.configure_git(vcs, self._mach_context.state_dir, self._mach_context.topdir) else: bootstrap.configure_mercurial(vcs, self._mach_context.state_dir)