def ensure_mobile_android_packages(self, mozconfig_builder, artifact_mode=False): java_bin_dir = self.ensure_java(mozconfig_builder) from mach.util import setenv setenv("PATH", "{}{}{}".format(java_bin_dir, os.pathsep, os.environ["PATH"])) from mozboot import android android.ensure_android( "windows", artifact_mode=artifact_mode, no_interactive=self.no_interactive )
def ensure_mobile_android_packages(self, mozconfig_builder, artifact_mode=False): java_path = get_oracle_jdk_8_path() or get_adopt_open_jdk_8_path() if java_path: from mach.util import setenv setenv('PATH', '{}{}{}'.format(os.path.join(java_path, 'bin'), os.pathsep, os.environ['PATH'])) self.ensure_java(mozconfig_builder) from mozboot import android android.ensure_android('windows', artifact_mode=artifact_mode, no_interactive=self.no_interactive)
def ensure_mobile_android_packages(self, artifact_mode=False): # Get java path from registry key try: import _winreg except ImportError: import winreg as _winreg key = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, r'SOFTWARE\JavaSoft\Java Development Kit\1.8') java_path, regtype = _winreg.QueryValueEx(key, 'JavaHome') _winreg.CloseKey(key) from mach.util import setenv setenv('PATH', '{}{}{}'.format(os.path.join(java_path, 'bin'), os.pathsep, os.environ['PATH'])) self.ensure_java() from mozboot import android android.ensure_android('windows', artifact_mode=artifact_mode, no_interactive=self.no_interactive)
def bootstrap(topsrcdir, mozilla_dir=None): if mozilla_dir is None: mozilla_dir = topsrcdir # Ensure we are running Python 2.7 or 3.5+. We put this check here so we # generate a user-friendly error message rather than a cryptic stack trace # on module import. major, minor = sys.version_info[:2] if (major == 2 and minor < 7) or (major == 3 and minor < 5): print('Python 2.7 or Python 3.5+ is required to run mach.') print('You are running Python', platform.python_version()) sys.exit(1) # Global build system and mach state is stored in a central directory. By # default, this is ~/.mozbuild. However, it can be defined via an # environment variable. We detect first run (by lack of this directory # existing) and notify the user that it will be created. The logic for # creation is much simpler for the "advanced" environment variable use # case. For default behavior, we educate users and give them an opportunity # to react. We always exit after creating the directory because users don't # like surprises. sys.path[0:0] = [os.path.join(mozilla_dir, path) for path in search_path(mozilla_dir, 'build/virtualenv_packages.txt')] import mach.base import mach.main from mach.util import setenv from mozboot.util import get_state_dir from mozbuild.util import patch_main patch_main() def resolve_repository(): import mozversioncontrol try: # This API doesn't respect the vcs binary choices from configure. # If we ever need to use the VCS binary here, consider something # more robust. return mozversioncontrol.get_repository_object(path=mozilla_dir) except (mozversioncontrol.InvalidRepoPath, mozversioncontrol.MissingVCSTool): return None def pre_dispatch_handler(context, handler, args): # If --disable-tests flag was enabled in the mozconfig used to compile # the build, tests will be disabled. Instead of trying to run # nonexistent tests then reporting a failure, this will prevent mach # from progressing beyond this point. if handler.category == 'testing': from mozbuild.base import BuildEnvironmentNotFoundException try: from mozbuild.base import MozbuildObject # all environments should have an instance of build object. build = MozbuildObject.from_environment() if build is not None and hasattr(build, 'mozconfig'): ac_options = build.mozconfig['configure_args'] if ac_options and '--disable-tests' in ac_options: print('Tests have been disabled by mozconfig with the flag ' + '"ac_add_options --disable-tests".\n' + 'Remove the flag, and re-compile to enable tests.') sys.exit(1) except BuildEnvironmentNotFoundException: # likely automation environment, so do nothing. pass def should_skip_telemetry_submission(handler): # The user is performing a maintenance command. if handler.name in ('bootstrap', 'doctor', 'mach-commands', 'vcs-setup', # We call mach environment in client.mk which would cause the # data submission to block the forward progress of make. 'environment'): return True # Never submit data when running in automation or when running tests. if any(e in os.environ for e in ('MOZ_AUTOMATION', 'TASK_ID', 'MACH_TELEMETRY_NO_SUBMIT')): return True return False def post_dispatch_handler(context, handler, instance, result, start_time, end_time, depth, args): """Perform global operations after command dispatch. For now, we will use this to handle build system telemetry. """ # Don't write telemetry data if this mach command was invoked as part of another # mach command. if depth != 1 or os.environ.get('MACH_MAIN_PID') != str(os.getpid()): return # Don't write telemetry data for 'mach' when 'DISABLE_TELEMETRY' is set. if os.environ.get('DISABLE_TELEMETRY') == '1': return # We have not opted-in to telemetry if not context.settings.build.telemetry: return from mozbuild.telemetry import gather_telemetry from mozbuild.base import MozbuildObject import mozpack.path as mozpath if not isinstance(instance, MozbuildObject): instance = MozbuildObject.from_environment() try: substs = instance.substs except Exception: substs = {} command_attrs = getattr(context, 'command_attrs', {}) # We gather telemetry for every operation. paths = { instance.topsrcdir: '$topsrcdir/', instance.topobjdir: '$topobjdir/', mozpath.normpath(os.path.expanduser('~')): '$HOME/', } # This might override one of the existing entries, that's OK. # We don't use a sigil here because we treat all arguments as potentially relative # paths, so we'd like to get them back as they were specified. paths[mozpath.normpath(os.getcwd())] = '' data = gather_telemetry(command=handler.name, success=(result == 0), start_time=start_time, end_time=end_time, mach_context=context, substs=substs, command_attrs=command_attrs, paths=paths) if data: telemetry_dir = os.path.join(get_state_dir(), 'telemetry') try: os.mkdir(telemetry_dir) except OSError as e: if e.errno != errno.EEXIST: raise outgoing_dir = os.path.join(telemetry_dir, 'outgoing') try: os.mkdir(outgoing_dir) except OSError as e: if e.errno != errno.EEXIST: raise with open(os.path.join(outgoing_dir, str(uuid.uuid4()) + '.json'), 'w') as f: json.dump(data, f, sort_keys=True) if should_skip_telemetry_submission(handler): return True state_dir = get_state_dir() machpath = os.path.join(instance.topsrcdir, 'mach') with open(os.devnull, 'wb') as devnull: subprocess.Popen([sys.executable, machpath, 'python', '--no-virtualenv', os.path.join(topsrcdir, 'build', 'submit_telemetry_data.py'), state_dir], stdout=devnull, stderr=devnull) def populate_context(context, key=None): if key is None: return if key == 'state_dir': state_dir = get_state_dir() if state_dir == os.environ.get('MOZBUILD_STATE_PATH'): if not os.path.exists(state_dir): print('Creating global state directory from environment variable: %s' % state_dir) os.makedirs(state_dir, mode=0o770) else: if not os.path.exists(state_dir): if not os.environ.get('MOZ_AUTOMATION'): print(STATE_DIR_FIRST_RUN.format(userdir=state_dir)) try: sys.stdin.readline() except KeyboardInterrupt: sys.exit(1) print('\nCreating default state directory: %s' % state_dir) os.makedirs(state_dir, mode=0o770) return state_dir if key == 'local_state_dir': return get_state_dir(srcdir=True) if key == 'topdir': return topsrcdir if key == 'pre_dispatch_handler': return pre_dispatch_handler if key == 'post_dispatch_handler': return post_dispatch_handler if key == 'repository': return resolve_repository() raise AttributeError(key) # Note which process is top-level so that recursive mach invocations can avoid writing # telemetry data. if 'MACH_MAIN_PID' not in os.environ: setenv('MACH_MAIN_PID', str(os.getpid())) driver = mach.main.Mach(os.getcwd()) driver.populate_context_handler = populate_context if not driver.settings_paths: # default global machrc location driver.settings_paths.append(get_state_dir()) # always load local repository configuration driver.settings_paths.append(mozilla_dir) for category, meta in CATEGORIES.items(): driver.define_category(category, meta['short'], meta['long'], meta['priority']) repo = resolve_repository() for path in MACH_MODULES: # Sparse checkouts may not have all mach_commands.py files. Ignore # errors from missing files. try: driver.load_commands_from_file(os.path.join(mozilla_dir, path)) except mach.base.MissingFileError: if not repo or not repo.sparse_checkout_present(): raise return driver
def initialize(topsrcdir): # Ensure we are running Python 3.6+. We run this check as soon as # possible to avoid a cryptic import/usage error. if sys.version_info < (3, 6): print("Python 3.6+ is required to run mach.") print("You are running Python", platform.python_version()) if sys.platform.startswith("linux"): print(INSTALL_PYTHON_GUIDANCE_LINUX) elif sys.platform.startswith("darwin"): print(INSTALL_PYTHON_GUIDANCE_OSX) elif "MOZILLABUILD" in os.environ: print(INSTALL_PYTHON_GUIDANCE_MOZILLABUILD) else: print(INSTALL_PYTHON_GUIDANCE_OTHER) sys.exit(1) # This directory was deleted in bug 1666345, but there may be some ignored # files here. We can safely just delete it for the user so they don't have # to clean the repo themselves. deleted_dir = os.path.join(topsrcdir, "third_party", "python", "psutil") if os.path.exists(deleted_dir): shutil.rmtree(deleted_dir, ignore_errors=True) state_dir = _create_state_dir() _activate_python_environment(topsrcdir) import mach.base import mach.main from mach.util import setenv from mozboot.util import get_state_dir # Set a reasonable limit to the number of open files. # # Some linux systems set `ulimit -n` to a very high number, which works # well for systems that run servers, but this setting causes performance # problems when programs close file descriptors before forking, like # Python's `subprocess.Popen(..., close_fds=True)` (close_fds=True is the # default in Python 3), or Rust's stdlib. In some cases, Firefox does the # same thing when spawning processes. We would prefer to lower this limit # to avoid such performance problems; processes spawned by `mach` will # inherit the limit set here. # # The Firefox build defaults the soft limit to 1024, except for builds that # do LTO, where the soft limit is 8192. We're going to default to the # latter, since people do occasionally do LTO builds on their local # machines, and requiring them to discover another magical setting after # setting up an LTO build in the first place doesn't seem good. # # This code mimics the code in taskcluster/scripts/run-task. try: import resource # Keep the hard limit the same, though, allowing processes to change # their soft limit if they need to (Firefox does, for instance). (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE) # Permit people to override our default limit if necessary via # MOZ_LIMIT_NOFILE, which is the same variable `run-task` uses. limit = os.environ.get("MOZ_LIMIT_NOFILE") if limit: limit = int(limit) else: # If no explicit limit is given, use our default if it's less than # the current soft limit. For instance, the default on macOS is # 256, so we'd pick that rather than our default. limit = min(soft, 8192) # Now apply the limit, if it's different from the original one. if limit != soft: resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) except ImportError: # The resource module is UNIX only. pass def resolve_repository(): import mozversioncontrol try: # This API doesn't respect the vcs binary choices from configure. # If we ever need to use the VCS binary here, consider something # more robust. return mozversioncontrol.get_repository_object(path=topsrcdir) except (mozversioncontrol.InvalidRepoPath, mozversioncontrol.MissingVCSTool): return None def pre_dispatch_handler(context, handler, args): # If --disable-tests flag was enabled in the mozconfig used to compile # the build, tests will be disabled. Instead of trying to run # nonexistent tests then reporting a failure, this will prevent mach # from progressing beyond this point. if handler.category == "testing" and not handler.ok_if_tests_disabled: from mozbuild.base import BuildEnvironmentNotFoundException try: from mozbuild.base import MozbuildObject # all environments should have an instance of build object. build = MozbuildObject.from_environment() if build is not None and not getattr(build, "substs", { "ENABLE_TESTS": True }).get("ENABLE_TESTS"): print("Tests have been disabled with --disable-tests.\n" + "Remove the flag, and re-compile to enable tests.") sys.exit(1) except BuildEnvironmentNotFoundException: # likely automation environment, so do nothing. pass def post_dispatch_handler(context, handler, instance, success, start_time, end_time, depth, args): """Perform global operations after command dispatch. For now, we will use this to handle build system telemetry. """ # Don't finalize telemetry data if this mach command was invoked as part of # another mach command. if depth != 1: return _finalize_telemetry_glean(context.telemetry, handler.name == "bootstrap", success) def populate_context(key=None): if key is None: return if key == "state_dir": return state_dir if key == "local_state_dir": return get_state_dir(srcdir=True) if key == "topdir": return topsrcdir if key == "pre_dispatch_handler": return pre_dispatch_handler if key == "post_dispatch_handler": return post_dispatch_handler if key == "repository": return resolve_repository() raise AttributeError(key) # Note which process is top-level so that recursive mach invocations can avoid writing # telemetry data. if "MACH_MAIN_PID" not in os.environ: setenv("MACH_MAIN_PID", str(os.getpid())) driver = mach.main.Mach(os.getcwd()) driver.populate_context_handler = populate_context if not driver.settings_paths: # default global machrc location driver.settings_paths.append(state_dir) # always load local repository configuration driver.settings_paths.append(topsrcdir) for category, meta in CATEGORIES.items(): driver.define_category(category, meta["short"], meta["long"], meta["priority"]) # Sparse checkouts may not have all mach_commands.py files. Ignore # errors from missing files. Same for spidermonkey tarballs. repo = resolve_repository() missing_ok = (repo is not None and repo.sparse_checkout_present()) or os.path.exists( os.path.join(topsrcdir, "INSTALL")) for path in MACH_MODULES: try: driver.load_commands_from_file(os.path.join(topsrcdir, path)) except mach.base.MissingFileError: if not missing_ok: raise return driver
def bootstrap(topsrcdir): # Ensure we are running Python 2.7 or 3.5+. We put this check here so we # generate a user-friendly error message rather than a cryptic stack trace # on module import. major, minor = sys.version_info[:2] if (major == 2 and minor < 7) or (major == 3 and minor < 5): print("Python 2.7 or Python 3.5+ is required to run mach.") print("You are running Python", platform.python_version()) sys.exit(1) # This directory was deleted in bug 1666345, but there may be some ignored # files here. We can safely just delete it for the user so they don't have # to clean the repo themselves. deleted_dir = os.path.join(topsrcdir, "third_party", "python", "psutil") if os.path.exists(deleted_dir): shutil.rmtree(deleted_dir, ignore_errors=True) if major == 3 and sys.prefix == sys.base_prefix: # We are not in a virtualenv. Remove global site packages # from sys.path. # Note that we don't ever invoke mach from a Python 2 virtualenv, # and "sys.base_prefix" doesn't exist before Python 3.3, so we # guard with the "major == 3" check. site_paths = set(site.getsitepackages() + [site.getusersitepackages()]) sys.path = [path for path in sys.path if path not in site_paths] # Global build system and mach state is stored in a central directory. By # default, this is ~/.mozbuild. However, it can be defined via an # environment variable. We detect first run (by lack of this directory # existing) and notify the user that it will be created. The logic for # creation is much simpler for the "advanced" environment variable use # case. For default behavior, we educate users and give them an opportunity # to react. We always exit after creating the directory because users don't # like surprises. sys.path[0:0] = mach_sys_path(topsrcdir) import mach.base import mach.main from mach.util import setenv from mozboot.util import get_state_dir # Set a reasonable limit to the number of open files. # # Some linux systems set `ulimit -n` to a very high number, which works # well for systems that run servers, but this setting causes performance # problems when programs close file descriptors before forking, like # Python's `subprocess.Popen(..., close_fds=True)` (close_fds=True is the # default in Python 3), or Rust's stdlib. In some cases, Firefox does the # same thing when spawning processes. We would prefer to lower this limit # to avoid such performance problems; processes spawned by `mach` will # inherit the limit set here. # # The Firefox build defaults the soft limit to 1024, except for builds that # do LTO, where the soft limit is 8192. We're going to default to the # latter, since people do occasionally do LTO builds on their local # machines, and requiring them to discover another magical setting after # setting up an LTO build in the first place doesn't seem good. # # This code mimics the code in taskcluster/scripts/run-task. try: import resource # Keep the hard limit the same, though, allowing processes to change # their soft limit if they need to (Firefox does, for instance). (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE) # Permit people to override our default limit if necessary via # MOZ_LIMIT_NOFILE, which is the same variable `run-task` uses. limit = os.environ.get("MOZ_LIMIT_NOFILE") if limit: limit = int(limit) else: # If no explicit limit is given, use our default if it's less than # the current soft limit. For instance, the default on macOS is # 256, so we'd pick that rather than our default. limit = min(soft, 8192) # Now apply the limit, if it's different from the original one. if limit != soft: resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) except ImportError: # The resource module is UNIX only. pass from mozbuild.util import patch_main patch_main() def resolve_repository(): import mozversioncontrol try: # This API doesn't respect the vcs binary choices from configure. # If we ever need to use the VCS binary here, consider something # more robust. return mozversioncontrol.get_repository_object(path=topsrcdir) except (mozversioncontrol.InvalidRepoPath, mozversioncontrol.MissingVCSTool): return None def pre_dispatch_handler(context, handler, args): # If --disable-tests flag was enabled in the mozconfig used to compile # the build, tests will be disabled. Instead of trying to run # nonexistent tests then reporting a failure, this will prevent mach # from progressing beyond this point. if handler.category == "testing" and not handler.ok_if_tests_disabled: from mozbuild.base import BuildEnvironmentNotFoundException try: from mozbuild.base import MozbuildObject # all environments should have an instance of build object. build = MozbuildObject.from_environment() if build is not None and hasattr(build, "mozconfig"): ac_options = build.mozconfig["configure_args"] if ac_options and "--disable-tests" in ac_options: print( "Tests have been disabled by mozconfig with the flag " + '"ac_add_options --disable-tests".\n' + "Remove the flag, and re-compile to enable tests.") sys.exit(1) except BuildEnvironmentNotFoundException: # likely automation environment, so do nothing. pass def post_dispatch_handler(context, handler, instance, success, start_time, end_time, depth, args): """Perform global operations after command dispatch. For now, we will use this to handle build system telemetry. """ # Don't finalize telemetry data if this mach command was invoked as part of # another mach command. if depth != 1: return _finalize_telemetry_glean(context.telemetry, handler.name == "bootstrap", success) def populate_context(key=None): if key is None: return if key == "state_dir": state_dir = get_state_dir() if state_dir == os.environ.get("MOZBUILD_STATE_PATH"): if not os.path.exists(state_dir): print( "Creating global state directory from environment variable: %s" % state_dir) os.makedirs(state_dir, mode=0o770) else: if not os.path.exists(state_dir): if not os.environ.get("MOZ_AUTOMATION"): print(STATE_DIR_FIRST_RUN.format(userdir=state_dir)) try: sys.stdin.readline() except KeyboardInterrupt: sys.exit(1) print("\nCreating default state directory: %s" % state_dir) os.makedirs(state_dir, mode=0o770) return state_dir if key == "local_state_dir": return get_state_dir(srcdir=True) if key == "topdir": return topsrcdir if key == "pre_dispatch_handler": return pre_dispatch_handler if key == "post_dispatch_handler": return post_dispatch_handler if key == "repository": return resolve_repository() raise AttributeError(key) # Note which process is top-level so that recursive mach invocations can avoid writing # telemetry data. if "MACH_MAIN_PID" not in os.environ: setenv("MACH_MAIN_PID", str(os.getpid())) driver = mach.main.Mach(os.getcwd()) driver.populate_context_handler = populate_context if not driver.settings_paths: # default global machrc location driver.settings_paths.append(get_state_dir()) # always load local repository configuration driver.settings_paths.append(topsrcdir) for category, meta in CATEGORIES.items(): driver.define_category(category, meta["short"], meta["long"], meta["priority"]) # Sparse checkouts may not have all mach_commands.py files. Ignore # errors from missing files. Same for spidermonkey tarballs. repo = resolve_repository() missing_ok = (repo is not None and repo.sparse_checkout_present()) or os.path.exists( os.path.join(topsrcdir, "INSTALL")) for path in MACH_MODULES: try: driver.load_commands_from_file(os.path.join(topsrcdir, path)) except mach.base.MissingFileError: if not missing_ok: raise return driver
def initialize(topsrcdir): # This directory was deleted in bug 1666345, but there may be some ignored # files here. We can safely just delete it for the user so they don't have # to clean the repo themselves. deleted_dir = os.path.join(topsrcdir, "third_party", "python", "psutil") if os.path.exists(deleted_dir): shutil.rmtree(deleted_dir, ignore_errors=True) # We need the "mach" module to access the logic to parse virtualenv # requirements. Since that depends on "packaging" (and, transitively, # "pyparsing"), we add those to the path too. sys.path[0:0] = [ os.path.join(topsrcdir, module) for module in ( os.path.join("python", "mach"), os.path.join("third_party", "python", "packaging"), os.path.join("third_party", "python", "pyparsing"), ) ] from mach.util import setenv, get_state_dir state_dir = _create_state_dir() # normpath state_dir to normalize msys-style slashes. _activate_python_environment( topsrcdir, lambda: os.path.normpath(get_state_dir(True, topsrcdir=topsrcdir)) ) _maybe_activate_mozillabuild_environment() import mach.base import mach.main from mach.main import MachCommandReference # Centralized registry of available mach commands MACH_COMMANDS = { "valgrind-test": MachCommandReference("build/valgrind/mach_commands.py"), "devtools-css-db": MachCommandReference( "devtools/shared/css/generated/mach_commands.py" ), "webidl-example": MachCommandReference("dom/bindings/mach_commands.py"), "webidl-parser-test": MachCommandReference("dom/bindings/mach_commands.py"), "hazards": MachCommandReference( "js/src/devtools/rootAnalysis/mach_commands.py" ), "reftest": MachCommandReference("layout/tools/reftest/mach_commands.py"), "jstestbrowser": MachCommandReference("layout/tools/reftest/mach_commands.py"), "crashtest": MachCommandReference("layout/tools/reftest/mach_commands.py"), "android": MachCommandReference("mobile/android/mach_commands.py"), "gradle": MachCommandReference("mobile/android/mach_commands.py"), "gradle-install": MachCommandReference("mobile/android/mach_commands.py"), "mach-commands": MachCommandReference( "python/mach/mach/commands/commandinfo.py" ), "mach-debug-commands": MachCommandReference( "python/mach/mach/commands/commandinfo.py" ), "mach-completion": MachCommandReference( "python/mach/mach/commands/commandinfo.py" ), "settings": MachCommandReference("python/mach/mach/commands/settings.py"), "python": MachCommandReference("python/mach_commands.py"), "python-test": MachCommandReference("python/mach_commands.py"), "bootstrap": MachCommandReference("python/mozboot/mozboot/mach_commands.py"), "vcs-setup": MachCommandReference("python/mozboot/mozboot/mach_commands.py"), "artifact": MachCommandReference( "python/mozbuild/mozbuild/artifact_commands.py" ), "ide": MachCommandReference( "python/mozbuild/mozbuild/backend/mach_commands.py" ), "build": MachCommandReference("python/mozbuild/mozbuild/build_commands.py"), "configure": MachCommandReference("python/mozbuild/mozbuild/build_commands.py"), "resource-usage": MachCommandReference( "python/mozbuild/mozbuild/build_commands.py" ), "build-backend": MachCommandReference( "python/mozbuild/mozbuild/build_commands.py" ), "clang-tidy": MachCommandReference( "python/mozbuild/mozbuild/code_analysis/mach_commands.py" ), "static-analysis": MachCommandReference( "python/mozbuild/mozbuild/code_analysis/mach_commands.py" ), "prettier-format": MachCommandReference( "python/mozbuild/mozbuild/code_analysis/mach_commands.py" ), "clang-format": MachCommandReference( "python/mozbuild/mozbuild/code_analysis/mach_commands.py" ), "compileflags": MachCommandReference( "python/mozbuild/mozbuild/compilation/codecomplete.py" ), "mozbuild-reference": MachCommandReference( "python/mozbuild/mozbuild/frontend/mach_commands.py" ), "file-info": MachCommandReference( "python/mozbuild/mozbuild/frontend/mach_commands.py" ), "vendor": MachCommandReference( "python/mozbuild/mozbuild/vendor/mach_commands.py" ), "watch": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), "cargo": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), "doctor": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), "clobber": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), "show-log": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), "warnings-summary": MachCommandReference( "python/mozbuild/mozbuild/mach_commands.py" ), "warnings-list": MachCommandReference( "python/mozbuild/mozbuild/mach_commands.py" ), "gtest": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), "package": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), "install": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), "run": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), "buildsymbols": MachCommandReference( "python/mozbuild/mozbuild/mach_commands.py" ), "environment": MachCommandReference( "python/mozbuild/mozbuild/mach_commands.py" ), "repackage": MachCommandReference("python/mozbuild/mozbuild/mach_commands.py"), "package-multi-locale": MachCommandReference( "python/mozbuild/mozbuild/mach_commands.py" ), "perftest": MachCommandReference( "python/mozperftest/mozperftest/mach_commands.py" ), "perftest-test": MachCommandReference( "python/mozperftest/mozperftest/mach_commands.py" ), "release": MachCommandReference( "python/mozrelease/mozrelease/mach_commands.py" ), "remote": MachCommandReference("remote/mach_commands.py"), "puppeteer-test": MachCommandReference("remote/mach_commands.py"), "generate-test-certs": MachCommandReference( "security/manager/tools/mach_commands.py" ), "taskgraph": MachCommandReference("taskcluster/mach_commands.py"), "taskcluster-load-image": MachCommandReference("taskcluster/mach_commands.py"), "taskcluster-build-image": MachCommandReference("taskcluster/mach_commands.py"), "taskcluster-image-digest": MachCommandReference( "taskcluster/mach_commands.py" ), "release-history": MachCommandReference("taskcluster/mach_commands.py"), "awsy-test": MachCommandReference("testing/awsy/mach_commands.py"), "fetch-condprofile": MachCommandReference( "testing/condprofile/mach_commands.py" ), "run-condprofile": MachCommandReference("testing/condprofile/mach_commands.py"), "firefox-ui-functional": MachCommandReference( "testing/firefox-ui/mach_commands.py" ), "geckodriver": MachCommandReference("testing/geckodriver/mach_commands.py"), "addtest": MachCommandReference("testing/mach_commands.py"), "test": MachCommandReference("testing/mach_commands.py"), "cppunittest": MachCommandReference("testing/mach_commands.py"), "jstests": MachCommandReference("testing/mach_commands.py"), "jit-test": MachCommandReference("testing/mach_commands.py"), "jsapi-tests": MachCommandReference("testing/mach_commands.py"), "jsshell-bench": MachCommandReference("testing/mach_commands.py"), "cramtest": MachCommandReference("testing/mach_commands.py"), "test-info": MachCommandReference("testing/mach_commands.py"), "rusttests": MachCommandReference("testing/mach_commands.py"), "fluent-migration-test": MachCommandReference("testing/mach_commands.py"), "marionette-test": MachCommandReference("testing/marionette/mach_commands.py"), "mochitest": MachCommandReference("testing/mochitest/mach_commands.py"), "geckoview-junit": MachCommandReference("testing/mochitest/mach_commands.py"), "mozharness": MachCommandReference("testing/mozharness/mach_commands.py"), "raptor": MachCommandReference("testing/raptor/mach_commands.py"), "raptor-test": MachCommandReference("testing/raptor/mach_commands.py"), "talos-test": MachCommandReference("testing/talos/mach_commands.py"), "tps-build": MachCommandReference("testing/tps/mach_commands.py"), "web-platform-tests": MachCommandReference( "testing/web-platform/mach_commands.py" ), "wpt": MachCommandReference("testing/web-platform/mach_commands.py"), "web-platform-tests-update": MachCommandReference( "testing/web-platform/mach_commands.py" ), "wpt-update": MachCommandReference("testing/web-platform/mach_commands.py"), "wpt-manifest-update": MachCommandReference( "testing/web-platform/mach_commands.py" ), "wpt-serve": MachCommandReference("testing/web-platform/mach_commands.py"), "wpt-metadata-summary": MachCommandReference( "testing/web-platform/mach_commands.py" ), "wpt-metadata-merge": MachCommandReference( "testing/web-platform/mach_commands.py" ), "wpt-unittest": MachCommandReference("testing/web-platform/mach_commands.py"), "wpt-test-paths": MachCommandReference("testing/web-platform/mach_commands.py"), "wpt-fission-regressions": MachCommandReference( "testing/web-platform/mach_commands.py" ), "xpcshell-test": MachCommandReference("testing/xpcshell/mach_commands.py"), "telemetry-tests-client": MachCommandReference( "toolkit/components/telemetry/tests/marionette/mach_commands.py" ), "data-review": MachCommandReference( "toolkit/components/glean/build_scripts/mach_commands.py" ), "update-glean-tags": MachCommandReference( "toolkit/components/glean/build_scripts/mach_commands.py" ), "browsertime": MachCommandReference("tools/browsertime/mach_commands.py"), "compare-locales": MachCommandReference( "tools/compare-locales/mach_commands.py" ), "l10n-cross-channel": MachCommandReference("tools/lint/mach_commands.py"), "busted": MachCommandReference("tools/mach_commands.py"), "pastebin": MachCommandReference("tools/mach_commands.py"), "mozregression": MachCommandReference("tools/mach_commands.py"), "node": MachCommandReference("tools/mach_commands.py"), "npm": MachCommandReference("tools/mach_commands.py"), "logspam": MachCommandReference("tools/mach_commands.py"), "doc": MachCommandReference("tools/moztreedocs/mach_commands.py"), "install-moz-phab": MachCommandReference("tools/phabricator/mach_commands.py"), "power": MachCommandReference("tools/power/mach_commands.py"), "try": MachCommandReference("tools/tryselect/mach_commands.py"), "import-pr": MachCommandReference("tools/vcs/mach_commands.py"), "test-interventions": MachCommandReference( "testing/webcompat/mach_commands.py" ), } # Set a reasonable limit to the number of open files. # # Some linux systems set `ulimit -n` to a very high number, which works # well for systems that run servers, but this setting causes performance # problems when programs close file descriptors before forking, like # Python's `subprocess.Popen(..., close_fds=True)` (close_fds=True is the # default in Python 3), or Rust's stdlib. In some cases, Firefox does the # same thing when spawning processes. We would prefer to lower this limit # to avoid such performance problems; processes spawned by `mach` will # inherit the limit set here. # # The Firefox build defaults the soft limit to 1024, except for builds that # do LTO, where the soft limit is 8192. We're going to default to the # latter, since people do occasionally do LTO builds on their local # machines, and requiring them to discover another magical setting after # setting up an LTO build in the first place doesn't seem good. # # This code mimics the code in taskcluster/scripts/run-task. try: import resource # Keep the hard limit the same, though, allowing processes to change # their soft limit if they need to (Firefox does, for instance). (soft, hard) = resource.getrlimit(resource.RLIMIT_NOFILE) # Permit people to override our default limit if necessary via # MOZ_LIMIT_NOFILE, which is the same variable `run-task` uses. limit = os.environ.get("MOZ_LIMIT_NOFILE") if limit: limit = int(limit) else: # If no explicit limit is given, use our default if it's less than # the current soft limit. For instance, the default on macOS is # 256, so we'd pick that rather than our default. limit = min(soft, 8192) # Now apply the limit, if it's different from the original one. if limit != soft: resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard)) except ImportError: # The resource module is UNIX only. pass def resolve_repository(): import mozversioncontrol try: # This API doesn't respect the vcs binary choices from configure. # If we ever need to use the VCS binary here, consider something # more robust. return mozversioncontrol.get_repository_object(path=topsrcdir) except (mozversioncontrol.InvalidRepoPath, mozversioncontrol.MissingVCSTool): return None def pre_dispatch_handler(context, handler, args): # If --disable-tests flag was enabled in the mozconfig used to compile # the build, tests will be disabled. Instead of trying to run # nonexistent tests then reporting a failure, this will prevent mach # from progressing beyond this point. if handler.category == "testing" and not handler.ok_if_tests_disabled: from mozbuild.base import BuildEnvironmentNotFoundException try: from mozbuild.base import MozbuildObject # all environments should have an instance of build object. build = MozbuildObject.from_environment() if build is not None and not getattr( build, "substs", {"ENABLE_TESTS": True} ).get("ENABLE_TESTS"): print( "Tests have been disabled with --disable-tests.\n" + "Remove the flag, and re-compile to enable tests." ) sys.exit(1) except BuildEnvironmentNotFoundException: # likely automation environment, so do nothing. pass def post_dispatch_handler( context, handler, instance, success, start_time, end_time, depth, args ): """Perform global operations after command dispatch. For now, we will use this to handle build system telemetry. """ # Don't finalize telemetry data if this mach command was invoked as part of # another mach command. if depth != 1: return _finalize_telemetry_glean( context.telemetry, handler.name == "bootstrap", success ) def populate_context(key=None): if key is None: return if key == "state_dir": return state_dir if key == "local_state_dir": return get_state_dir(specific_to_topsrcdir=True) if key == "topdir": return topsrcdir if key == "pre_dispatch_handler": return pre_dispatch_handler if key == "post_dispatch_handler": return post_dispatch_handler if key == "repository": return resolve_repository() raise AttributeError(key) # Note which process is top-level so that recursive mach invocations can avoid writing # telemetry data. if "MACH_MAIN_PID" not in os.environ: setenv("MACH_MAIN_PID", str(os.getpid())) driver = mach.main.Mach(os.getcwd()) driver.populate_context_handler = populate_context if not driver.settings_paths: # default global machrc location driver.settings_paths.append(state_dir) # always load local repository configuration driver.settings_paths.append(topsrcdir) for category, meta in CATEGORIES.items(): driver.define_category(category, meta["short"], meta["long"], meta["priority"]) # Sparse checkouts may not have all mach_commands.py files. Ignore # errors from missing files. Same for spidermonkey tarballs. repo = resolve_repository() missing_ok = ( repo is not None and repo.sparse_checkout_present() ) or os.path.exists(os.path.join(topsrcdir, "INSTALL")) driver.load_commands_from_spec(MACH_COMMANDS, topsrcdir, missing_ok=missing_ok) return driver