def build_shell(self, **kwargs): '''Build a JS shell to use to run the rooting hazard analysis.''' # The JS shell requires some specific configuration settings to execute # the hazard analysis code, and configuration is done via mozconfig. # Subprocesses find MOZCONFIG in the environment, so we can't just # modify the settings in this process's loaded version. Pass it through # the environment. default_mozconfig = 'js/src/devtools/rootAnalysis/mozconfig.haz_shell' mozconfig_path = kwargs.pop('mozconfig', None) \ or os.environ.get('MOZCONFIG') \ or default_mozconfig mozconfig_path = os.path.join(self.topsrcdir, mozconfig_path) loader = MozconfigLoader(self.topsrcdir) mozconfig = loader.read_mozconfig(mozconfig_path) # Validate the mozconfig settings in case the user overrode the default. configure_args = mozconfig['configure_args'] if '--enable-ctypes' not in configure_args: raise FailedCommandError('ctypes required in hazard JS shell') # Transmit the mozconfig location to build subprocesses. os.environ['MOZCONFIG'] = mozconfig_path # Record the location of the JS shell in the analysis work dir. self.write_json_file( "shell.json", {'js': os.path.join(mozconfig['topobjdir'], "dist/bin/js")}) return self._mach_context.commands.dispatch('build', self._mach_context, **kwargs)
def build_shell(self, command_context, **kwargs): """Build a JS shell to use to run the rooting hazard analysis.""" # The JS shell requires some specific configuration settings to execute # the hazard analysis code, and configuration is done via mozconfig. # Subprocesses find MOZCONFIG in the environment, so we can't just # modify the settings in this process's loaded version. Pass it through # the environment. default_mozconfig = "js/src/devtools/rootAnalysis/mozconfig.haz_shell" mozconfig_path = (kwargs.pop("mozconfig", None) or os.environ.get("MOZCONFIG") or default_mozconfig) mozconfig_path = os.path.join(self.topsrcdir, mozconfig_path) loader = MozconfigLoader(self.topsrcdir) mozconfig = loader.read_mozconfig(mozconfig_path) # Validate the mozconfig settings in case the user overrode the default. configure_args = mozconfig["configure_args"] if "--enable-ctypes" not in configure_args: raise FailedCommandError( "ctypes required in hazard JS shell, mozconfig=" + mozconfig_path) # Transmit the mozconfig location to build subprocesses. os.environ["MOZCONFIG"] = mozconfig_path self.setup_env_for_tools(os.environ) # Set a default objdir for the shell, for developer builds. os.environ.setdefault("MOZ_OBJDIR", os.path.join(self.topsrcdir, "obj-haz-shell")) return self._mach_context.commands.dispatch("build", self._mach_context, **kwargs)
def test_find_multiple_configs(self): """Ensure multiple relative-path MOZCONFIGs result in error.""" relative_mozconfig = '.mconfig' os.environ[b'MOZCONFIG'] = relative_mozconfig srcdir = self.get_temp_dir() curdir = self.get_temp_dir() dirs = [srcdir, curdir] loader = MozconfigLoader(srcdir) for d in dirs: path = os.path.join(d, relative_mozconfig) with open(path, 'wb') as f: f.write(path) orig_dir = os.getcwd() try: os.chdir(curdir) with self.assertRaises(MozconfigFindException) as e: loader.find_mozconfig() finally: os.chdir(orig_dir) self.assertIn('exists in more than one of', e.exception.message) for d in dirs: self.assertIn(d, e.exception.message)
def autoconf_refresh(configure): if os.path.exists(configure): mtime = os.path.getmtime(configure) aclocal = os.path.join(base_dir, 'build', 'autoconf', '*.m4') for input in itertools.chain( (configure + '.in', os.path.join(os.path.dirname(configure), 'aclocal.m4')), glob.iglob(aclocal), ): if os.path.getmtime(input) > mtime: break else: return mozconfig_autoconf = None configure_dir = os.path.dirname(configure) # Don't read the mozconfig for the js configure (yay backwards # compatibility) if not configure_dir.replace(os.sep, '/').endswith('/js/src'): loader = MozconfigLoader(os.path.dirname(configure)) project = os.environ.get('MOZ_CURRENT_PROJECT') mozconfig = loader.find_mozconfig(env=os.environ) mozconfig = loader.read_mozconfig(mozconfig, moz_build_app=project) make_extra = mozconfig['make_extra'] if make_extra: for assignment in make_extra: m = re.match('(?:export\s+)?AUTOCONF\s*:?=\s*(.+)$', assignment) if m: mozconfig_autoconf = m.group(1) for ac in (mozconfig_autoconf, os.environ.get('AUTOCONF'), 'autoconf-2.13', 'autoconf2.13', 'autoconf213'): if ac: autoconf = find_program(ac) if autoconf: break else: fink = find_program('fink') if fink: autoconf = os.path.normpath( os.path.join(fink, '..', '..', 'lib', 'autoconf2.13', 'bin', 'autoconf')) if not autoconf: raise RuntimeError('Could not find autoconf 2.13') # Add or adjust AUTOCONF for subprocesses, especially the js/src configure os.environ['AUTOCONF'] = autoconf print('Refreshing %s with %s' % (configure, autoconf), file=sys.stderr) with open(configure, 'wb') as fh: subprocess.check_call([ shell, autoconf, '--localdir=%s' % os.path.dirname(configure), configure + '.in' ], stdout=fh)
def test_find_default_files(self): """Ensure default paths are used when present.""" for p in MozconfigLoader.DEFAULT_TOPSRCDIR_PATHS: d = self.get_temp_dir() path = os.path.join(d, p) with open(path, 'w'): pass self.assertEqual(MozconfigLoader(d).find_mozconfig(), path)
def inner_compile(self, **kwargs): '''Build a source tree and gather analysis information while running under the influence of the analysis collection server.''' env = os.environ # Check whether we are running underneath the manager (and therefore # have a server to talk to). if 'XGILL_CONFIG' not in env: raise Exception( 'no sixgill manager detected. `mach hazards compile` ' + 'should only be run from `mach hazards gather`' ) app = kwargs.pop('application') self.check_application(app) default_mozconfig = 'js/src/devtools/rootAnalysis/mozconfig.%s' % app mozconfig_path = kwargs.pop('mozconfig', None) \ or env.get('MOZCONFIG') \ or default_mozconfig mozconfig_path = os.path.join(self.topsrcdir, mozconfig_path) # Validate the mozconfig. # Require an explicit --enable-application=APP (even if you just # want to build the default browser application.) loader = MozconfigLoader(self.topsrcdir) mozconfig = loader.read_mozconfig(mozconfig_path) configure_args = mozconfig['configure_args'] if '--enable-application=%s' % app not in configure_args: raise Exception('mozconfig %s builds wrong project' % mozconfig_path) if not any('--with-compiler-wrapper' in a for a in configure_args): raise Exception('mozconfig must wrap compiles') # Communicate mozconfig to build subprocesses. env['MOZCONFIG'] = os.path.join(self.topsrcdir, mozconfig_path) # hazard mozconfigs need to find binaries in .mozbuild env['MOZBUILD_STATE_PATH'] = self.state_dir # Force the use of hazard-compatible installs of tools. gccbin = os.path.join(self.gcc_dir, 'bin') env['CC'] = os.path.join(gccbin, 'gcc') env['CXX'] = os.path.join(gccbin, 'g++') env['PATH'] = '{sixgill_dir}/usr/bin:{gccbin}:{PATH}'.format( sixgill_dir=self.sixgill_dir, gccbin=gccbin, PATH=env['PATH'] ) env['LD_LIBRARY_PATH'] = '{}/lib64'.format(self.gcc_dir) return self._mach_context.commands.dispatch( 'build', self._mach_context, **kwargs )
def _get_mozconfig_env(self, config): env = {} loader = MozconfigLoader(config.topsrcdir) mozconfig = loader.read_mozconfig(config.substs['MOZCONFIG']) make_extra = mozconfig['make_extra'] or [] env = {} for line in make_extra: if line.startswith('export '): line = line[len('export '):] key, value = line.split('=') env[key] = value return env
def test_find_deprecated_path_srcdir(self): """Ensure we error when deprecated path locations are present.""" for p in MozconfigLoader.DEPRECATED_TOPSRCDIR_PATHS: d = self.get_temp_dir() with open(os.path.join(d, p), 'w'): pass with self.assertRaises(MozconfigFindException) as e: MozconfigLoader(d).find_mozconfig() self.assertIn('This implicit location is no longer', e.exception.message) self.assertIn(d, e.exception.message)
def test_find_multiple_defaults(self): """Ensure we error when multiple default files are present.""" self.assertGreater(len(MozconfigLoader.DEFAULT_TOPSRCDIR_PATHS), 1) d = self.get_temp_dir() for p in MozconfigLoader.DEFAULT_TOPSRCDIR_PATHS: with open(os.path.join(d, p), 'w'): pass with self.assertRaises(MozconfigFindException) as e: MozconfigLoader(d).find_mozconfig() self.assertIn('Multiple default mozconfig files present', e.exception.message)
def inner_compile(self, **kwargs): """Build a source tree and gather analysis information while running under the influence of the analysis collection server.""" env = os.environ # Check whether we are running underneath the manager (and therefore # have a server to talk to). if "XGILL_CONFIG" not in env: raise Exception( "no sixgill manager detected. `mach hazards compile` " + "should only be run from `mach hazards gather`" ) app = kwargs.pop("application") default_mozconfig = "js/src/devtools/rootAnalysis/mozconfig.%s" % app mozconfig_path = ( kwargs.pop("mozconfig", None) or env.get("MOZCONFIG") or default_mozconfig ) mozconfig_path = os.path.join(self.topsrcdir, mozconfig_path) # Validate the mozconfig. # Require an explicit --enable-application=APP (even if you just # want to build the default browser application.) loader = MozconfigLoader(self.topsrcdir) mozconfig = loader.read_mozconfig(mozconfig_path) configure_args = mozconfig["configure_args"] if "--enable-application=%s" % app not in configure_args: raise Exception("mozconfig %s builds wrong project" % mozconfig_path) if not any("--with-compiler-wrapper" in a for a in configure_args): raise Exception("mozconfig must wrap compiles") # Communicate mozconfig to build subprocesses. env["MOZCONFIG"] = os.path.join(self.topsrcdir, mozconfig_path) # hazard mozconfigs need to find binaries in .mozbuild env["MOZBUILD_STATE_PATH"] = self.state_dir # Suppress the gathering of sources, to save disk space and memory. env["XGILL_NO_SOURCE"] = "1" self.setup_env_for_tools(env) if "haz_objdir" in kwargs: env["MOZ_OBJDIR"] = kwargs.pop("haz_objdir") return self._mach_context.commands.dispatch( "build", self._mach_context, **kwargs )
def inner_compile(self, **kwargs): """Build a source tree and gather analysis information while running under the influence of the analysis collection server.""" env = os.environ # Check whether we are running underneath the manager (and therefore # have a server to talk to). if "XGILL_CONFIG" not in env: raise Exception( "no sixgill manager detected. `mach hazards compile` " + "should only be run from `mach hazards gather`") app = kwargs.pop("application") self.check_application(app) default_mozconfig = "js/src/devtools/rootAnalysis/mozconfig.%s" % app mozconfig_path = (kwargs.pop("mozconfig", None) or env.get("MOZCONFIG") or default_mozconfig) mozconfig_path = os.path.join(self.topsrcdir, mozconfig_path) # Validate the mozconfig. # Require an explicit --enable-application=APP (even if you just # want to build the default browser application.) loader = MozconfigLoader(self.topsrcdir) mozconfig = loader.read_mozconfig(mozconfig_path) configure_args = mozconfig["configure_args"] if "--enable-application=%s" % app not in configure_args: raise Exception("mozconfig %s builds wrong project" % mozconfig_path) if not any("--with-compiler-wrapper" in a for a in configure_args): raise Exception("mozconfig must wrap compiles") # Communicate mozconfig to build subprocesses. env["MOZCONFIG"] = os.path.join(self.topsrcdir, mozconfig_path) # hazard mozconfigs need to find binaries in .mozbuild env["MOZBUILD_STATE_PATH"] = self.state_dir # Force the use of hazard-compatible installs of tools. gccbin = os.path.join(self.gcc_dir, "bin") env["CC"] = os.path.join(gccbin, "gcc") env["CXX"] = os.path.join(gccbin, "g++") env["PATH"] = "{sixgill_dir}/usr/bin:{gccbin}:{PATH}".format( sixgill_dir=self.sixgill_dir, gccbin=gccbin, PATH=env["PATH"]) env["LD_LIBRARY_PATH"] = "{}/lib64".format(self.gcc_dir) return self._mach_context.commands.dispatch("build", self._mach_context, **kwargs)
def test_find_relative_mozconfig(self): """Ensure a relative MOZCONFIG can be found in the srcdir.""" relative_mozconfig = '.mconfig' os.environ[b'MOZCONFIG'] = relative_mozconfig srcdir = self.get_temp_dir() curdir = self.get_temp_dir() loader = MozconfigLoader(srcdir) path = os.path.join(srcdir, relative_mozconfig) with open(path, 'w'): pass orig_dir = os.getcwd() try: os.chdir(curdir) self.assertEqual(os.path.normpath(loader.find_mozconfig()), os.path.normpath(path)) finally: os.chdir(orig_dir)
def test_find_no_relative_configs(self): """Ensure a missing relative-path MOZCONFIG is detected.""" relative_mozconfig = '.mconfig' os.environ[b'MOZCONFIG'] = relative_mozconfig srcdir = self.get_temp_dir() curdir = self.get_temp_dir() dirs = [srcdir, curdir] loader = MozconfigLoader(srcdir) orig_dir = os.getcwd() try: os.chdir(curdir) with self.assertRaises(MozconfigFindException) as e: loader.find_mozconfig() finally: os.chdir(orig_dir) self.assertIn('does not exist in any of', e.exception.message) for d in dirs: self.assertIn(d, e.exception.message)
def test_find_multiple_but_identical_configs(self): """Ensure multiple relative-path MOZCONFIGs pointing at the same file are OK.""" relative_mozconfig = '../src/.mconfig' os.environ[b'MOZCONFIG'] = relative_mozconfig topdir = self.get_temp_dir() srcdir = os.path.join(topdir, 'src') os.mkdir(srcdir) curdir = os.path.join(topdir, 'obj') os.mkdir(curdir) loader = MozconfigLoader(srcdir) path = os.path.join(srcdir, relative_mozconfig) with open(path, 'w'): pass orig_dir = os.getcwd() try: os.chdir(curdir) self.assertEqual(os.path.realpath(loader.find_mozconfig()), os.path.realpath(path)) finally: os.chdir(orig_dir)
def get_loader(self): return MozconfigLoader(self.get_temp_dir())
def build(self, what=None, disable_extra_make_dependencies=None, jobs=0, directory=None, verbose=False, keep_going=False): """Build the source tree. With no arguments, this will perform a full build. Positional arguments define targets to build. These can be make targets or patterns like "<dir>/<target>" to indicate a make target within a directory. There are a few special targets that can be used to perform a partial build faster than what `mach build` would perform: * binaries - compiles and links all C/C++ sources and produces shared libraries and executables (binaries). * faster - builds JavaScript, XUL, CSS, etc files. "binaries" and "faster" almost fully complement each other. However, there are build actions not captured by either. If things don't appear to be rebuilding, perform a vanilla `mach build` to rebuild the world. """ from mozbuild.controller.building import ( BuildDriver, ) self.log_manager.enable_all_structured_loggers() loader = MozconfigLoader(self.topsrcdir) mozconfig = loader.read_mozconfig(loader.AUTODETECT) configure_args = mozconfig['configure_args'] doing_pgo = configure_args and 'MOZ_PGO=1' in configure_args append_env = None if doing_pgo: if what: raise Exception('Cannot specify targets (%s) in MOZ_PGO=1 builds' % what) instr = self._spawn(BuildDriver) orig_topobjdir = instr._topobjdir instr._topobjdir = mozpath.join(instr._topobjdir, 'instrumented') append_env = {'MOZ_PROFILE_GENERATE': '1'} status = instr.build( what=what, disable_extra_make_dependencies=disable_extra_make_dependencies, jobs=jobs, directory=directory, verbose=verbose, keep_going=keep_going, mach_context=self._mach_context, append_env=append_env) if status != 0: return status # Packaging the instrumented build is required to get the jarlog # data. status = instr._run_make( directory=".", target='package', silent=not verbose, ensure_exit_code=False, append_env=append_env) if status != 0: return status pgo_env = os.environ.copy() pgo_env['LLVM_PROFDATA'] = instr.config_environment.substs.get('LLVM_PROFDATA') pgo_env['JARLOG_FILE'] = mozpath.join(orig_topobjdir, 'jarlog/en-US.log') pgo_cmd = [ instr.virtualenv_manager.python_path, mozpath.join(self.topsrcdir, 'build/pgo/profileserver.py'), ] subprocess.check_call(pgo_cmd, cwd=instr.topobjdir, env=ensure_subprocess_env(pgo_env)) # Set the default build to MOZ_PROFILE_USE append_env = {'MOZ_PROFILE_USE': '1'} driver = self._spawn(BuildDriver) return driver.build( what=what, disable_extra_make_dependencies=disable_extra_make_dependencies, jobs=jobs, directory=directory, verbose=verbose, keep_going=keep_going, mach_context=self._mach_context, append_env=append_env)
def build( command_context, what=None, jobs=0, job_size=0, directory=None, verbose=False, keep_going=False, priority="less", ): """Build the source tree. With no arguments, this will perform a full build. Positional arguments define targets to build. These can be make targets or patterns like "<dir>/<target>" to indicate a make target within a directory. There are a few special targets that can be used to perform a partial build faster than what `mach build` would perform: * binaries - compiles and links all C/C++ sources and produces shared libraries and executables (binaries). * faster - builds JavaScript, XUL, CSS, etc files. "binaries" and "faster" almost fully complement each other. However, there are build actions not captured by either. If things don't appear to be rebuilding, perform a vanilla `mach build` to rebuild the world. """ from mozbuild.controller.building import BuildDriver command_context.log_manager.enable_all_structured_loggers() loader = MozconfigLoader(command_context.topsrcdir) mozconfig = loader.read_mozconfig(loader.AUTODETECT) configure_args = mozconfig["configure_args"] doing_pgo = configure_args and "MOZ_PGO=1" in configure_args # Force verbosity on automation. verbose = verbose or bool(os.environ.get("MOZ_AUTOMATION", False)) append_env = None # By setting the current process's priority, by default our child processes # will also inherit this same priority. if not _set_priority(priority, verbose): print("--priority not supported on this platform.") if doing_pgo: if what: raise Exception("Cannot specify targets (%s) in MOZ_PGO=1 builds" % what) instr = command_context._spawn(BuildDriver) orig_topobjdir = instr._topobjdir instr._topobjdir = mozpath.join(instr._topobjdir, "instrumented") append_env = {"MOZ_PROFILE_GENERATE": "1"} status = instr.build( command_context.metrics, what=what, jobs=jobs, job_size=job_size, directory=directory, verbose=verbose, keep_going=keep_going, mach_context=command_context._mach_context, append_env=append_env, ) if status != 0: return status # Packaging the instrumented build is required to get the jarlog # data. status = instr._run_make( directory=".", target="package", silent=not verbose, ensure_exit_code=False, append_env=append_env, ) if status != 0: return status pgo_env = os.environ.copy() if instr.config_environment.substs.get("CC_TYPE") in ("clang", "clang-cl"): pgo_env["LLVM_PROFDATA"] = instr.config_environment.substs.get( "LLVM_PROFDATA") pgo_env["JARLOG_FILE"] = mozpath.join(orig_topobjdir, "jarlog/en-US.log") pgo_cmd = [ instr.virtualenv_manager.python_path, mozpath.join(command_context.topsrcdir, "build/pgo/profileserver.py"), ] subprocess.check_call(pgo_cmd, cwd=instr.topobjdir, env=pgo_env) # Set the default build to MOZ_PROFILE_USE append_env = {"MOZ_PROFILE_USE": "1"} driver = command_context._spawn(BuildDriver) return driver.build( command_context.metrics, what=what, jobs=jobs, job_size=job_size, directory=directory, verbose=verbose, keep_going=keep_going, mach_context=command_context._mach_context, append_env=append_env, )