def test_add_extra_dependencies(self): # a depends on b depends on c depends on d dm = {"a": ["b", "c", "d"], "b": ["c", "d"], "c": ["d"]} # Edge cases. self.assertEqual(list(add_extra_dependencies([], dependency_map=dm)), []) self.assertEqual(list(add_extra_dependencies([(None, "package")], dependency_map=dm)), [(None, "package")]) # Easy expansion. self.assertEqual(list(add_extra_dependencies([("b", None)], dependency_map=dm)), [("b", None), ("c", None), ("d", None)]) # Expansion with two groups -- each group is handled independently. self.assertEqual(list(add_extra_dependencies([("b", None), (None, "package"), ("c", None)], dependency_map=dm)), [("b", None), ("c", None), ("d", None), (None, "package"), ("c", None), ("d", None)]) # Two groups, no duplicate dependencies in each group. self.assertEqual(list(add_extra_dependencies([("a", None), ("b", None), (None, "package"), (None, "install"), ("c", None), ("d", None)], dependency_map=dm)), [("a", None), ("b", None), ("c", None), ("d", None), (None, "package"), (None, "install"), ("c", None), ("d", None)])
def test_add_extra_dependencies(self): # a depends on b depends on c depends on d dm = {"a": ["b", "c", "d"], "b": ["c", "d"], "c": ["d"]} # Edge cases. self.assertEqual(list(add_extra_dependencies([], dependency_map=dm)), []) self.assertEqual( list(add_extra_dependencies([(None, "package")], dependency_map=dm)), [(None, "package")]) # Easy expansion. self.assertEqual( list(add_extra_dependencies([("b", None)], dependency_map=dm)), [("b", None), ("c", None), ("d", None)]) # Expansion with two groups -- each group is handled independently. self.assertEqual( list( add_extra_dependencies([("b", None), (None, "package"), ("c", None)], dependency_map=dm)), [("b", None), (None, "package"), ("c", None), ("d", None)]) # Two groups, no duplicate dependencies in each group. self.assertEqual( list( add_extra_dependencies([("a", None), ("b", None), (None, "package"), (None, "install"), ("c", None), ("d", None)], dependency_map=dm)), [("a", None), ("b", None), (None, "package"), (None, "install"), ("c", None), ("d", None)])
def build(self, what=None, disable_extra_make_dependencies=None, jobs=0, verbose=False): from mozbuild.controller.building import BuildMonitor from mozbuild.util import resolve_target_to_make warnings_path = self._get_state_filename('warnings.json') monitor = BuildMonitor(self.topobjdir, warnings_path) with BuildOutputManager(self.log_manager, monitor) as output: monitor.start() if what: top_make = os.path.join(self.topobjdir, 'Makefile') if not os.path.exists(top_make): print('Your tree has not been configured yet. Please run ' '|mach build| with no arguments.') return 1 # Collect target pairs. target_pairs = [] for target in what: path_arg = self._wrap_path_argument(target) make_dir, make_target = resolve_target_to_make(self.topobjdir, path_arg.relpath()) if make_dir is None and make_target is None: return 1 target_pairs.append((make_dir, make_target)) # Possibly add extra make depencies using dumbmake. if not disable_extra_make_dependencies: from dumbmake.dumbmake import (dependency_map, add_extra_dependencies) depfile = os.path.join(self.topsrcdir, 'build', 'dumbmake-dependencies') with open(depfile) as f: dm = dependency_map(f.readlines()) new_pairs = list(add_extra_dependencies(target_pairs, dm)) self.log(logging.DEBUG, 'dumbmake', {'target_pairs': target_pairs, 'new_pairs': new_pairs}, 'Added extra dependencies: will build {new_pairs} ' + 'instead of {target_pairs}.') target_pairs = new_pairs # Build target pairs. for make_dir, make_target in target_pairs: status = self._run_make(directory=make_dir, target=make_target, line_handler=output.on_line, log=False, print_directory=False, ensure_exit_code=False, num_jobs=jobs, silent=not verbose) if status != 0: break else: status = self._run_make(srcdir=True, filename='client.mk', line_handler=output.on_line, log=False, print_directory=False, allow_parallel=False, ensure_exit_code=False, num_jobs=jobs, silent=not verbose) self.log(logging.WARNING, 'warning_summary', {'count': len(monitor.warnings_database)}, '{count} compiler warnings present.') monitor.finish() high_finder, finder_percent = monitor.have_high_finder_usage() if high_finder: print(FINDER_SLOW_MESSAGE % finder_percent) long_build = monitor.elapsed > 600 if status: return status if long_build: print('We know it took a while, but your build finally finished successfully!') else: print('Your build was successful!') # Only for full builds because incremental builders likely don't # need to be burdened with this. if not what: # Fennec doesn't have useful output from just building. We should # arguably make the build action useful for Fennec. Another day... if self.substs['MOZ_BUILD_APP'] != 'mobile/android': app_path = self.get_binary_path('app') print('To take your build for a test drive, run: %s' % app_path) app = self.substs['MOZ_BUILD_APP'] if app in ('browser', 'mobile/android'): print('For more information on what to do now, see ' 'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox') return status
def build(self, what=None, disable_extra_make_dependencies=None, jobs=0): # This code is only meant to be temporary until the more robust tree # building code in bug 780329 lands. from mozbuild.compilation.warnings import WarningsCollector from mozbuild.compilation.warnings import WarningsDatabase from mozbuild.util import resolve_target_to_make warnings_path = self._get_state_filename('warnings.json') warnings_database = WarningsDatabase() if os.path.exists(warnings_path): try: warnings_database.load_from_file(warnings_path) except ValueError: os.remove(warnings_path) warnings_collector = WarningsCollector(database=warnings_database, objdir=self.topobjdir) def on_line(line): try: warning = warnings_collector.process_line(line) if warning: self.log(logging.INFO, 'compiler_warning', warning, 'Warning: {flag} in {filename}: {message}') except: # This will get logged in the more robust implementation. pass self.log(logging.INFO, 'build_output', {'line': line}, '{line}') finder_start_cpu = self._get_finder_cpu_usage() time_start = time.time() if what: top_make = os.path.join(self.topobjdir, 'Makefile') if not os.path.exists(top_make): print('Your tree has not been configured yet. Please run ' '|mach build| with no arguments.') return 1 # Collect target pairs. target_pairs = [] for target in what: path_arg = self._wrap_path_argument(target) make_dir, make_target = resolve_target_to_make(self.topobjdir, path_arg.relpath()) if make_dir is None and make_target is None: return 1 target_pairs.append((make_dir, make_target)) # Possibly add extra make depencies using dumbmake. if not disable_extra_make_dependencies: from dumbmake.dumbmake import (dependency_map, add_extra_dependencies) depfile = os.path.join(self.topsrcdir, 'build', 'dumbmake-dependencies') with open(depfile) as f: dm = dependency_map(f.readlines()) new_pairs = list(add_extra_dependencies(target_pairs, dm)) self.log(logging.DEBUG, 'dumbmake', {'target_pairs': target_pairs, 'new_pairs': new_pairs}, 'Added extra dependencies: will build {new_pairs} ' + 'instead of {target_pairs}.') target_pairs = new_pairs # Build target pairs. for make_dir, make_target in target_pairs: status = self._run_make(directory=make_dir, target=make_target, line_handler=on_line, log=False, print_directory=False, ensure_exit_code=False, num_jobs=jobs) if status != 0: break else: status = self._run_make(srcdir=True, filename='client.mk', line_handler=on_line, log=False, print_directory=False, allow_parallel=False, ensure_exit_code=False, num_jobs=jobs) self.log(logging.WARNING, 'warning_summary', {'count': len(warnings_collector.database)}, '{count} compiler warnings present.') warnings_database.prune() warnings_database.save_to_file(warnings_path) time_end = time.time() time_elapsed = time_end - time_start self._handle_finder_cpu_usage(time_elapsed, finder_start_cpu) long_build = time_elapsed > 600 if status: return status if long_build: print('We know it took a while, but your build finally finished successfully!') else: print('Your build was successful!') # Only for full builds because incremental builders likely don't # need to be burdened with this. if not what: # Fennec doesn't have useful output from just building. We should # arguably make the build action useful for Fennec. Another day... if self.substs['MOZ_BUILD_APP'] != 'mobile/android': app_path = self.get_binary_path('app') print('To take your build for a test drive, run: %s' % app_path) app = self.substs['MOZ_BUILD_APP'] if app in ('browser', 'mobile/android'): print('For more information on what to do now, see ' 'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox') return status
def build(self, what=None, disable_extra_make_dependencies=None, jobs=0, directory=None, verbose=False): import which from mozbuild.controller.building import BuildMonitor from mozbuild.util import resolve_target_to_make self.log_manager.register_structured_logger(logging.getLogger('mozbuild')) warnings_path = self._get_state_filename('warnings.json') monitor = self._spawn(BuildMonitor) monitor.init(warnings_path) ccache_start = monitor.ccache_stats() with BuildOutputManager(self.log_manager, monitor) as output: monitor.start() if directory is not None and not what: print('Can only use -C/--directory with an explicit target ' 'name.') return 1 if directory is not None: disable_extra_make_dependencies=True directory = mozpath.normsep(directory) if directory.startswith('/'): directory = directory[1:] if what: top_make = os.path.join(self.topobjdir, 'Makefile') if not os.path.exists(top_make): print('Your tree has not been configured yet. Please run ' '|mach build| with no arguments.') return 1 # Collect target pairs. target_pairs = [] for target in what: path_arg = self._wrap_path_argument(target) if directory is not None: make_dir = os.path.join(self.topobjdir, directory) make_target = target else: make_dir, make_target = \ resolve_target_to_make(self.topobjdir, path_arg.relpath()) if make_dir is None and make_target is None: return 1 # See bug 886162 - we don't want to "accidentally" build # the entire tree (if that's really the intent, it's # unlikely they would have specified a directory.) if not make_dir and not make_target: print("The specified directory doesn't contain a " "Makefile and the first parent with one is the " "root of the tree. Please specify a directory " "with a Makefile or run |mach build| if you " "want to build the entire tree.") return 1 target_pairs.append((make_dir, make_target)) # Possibly add extra make depencies using dumbmake. if not disable_extra_make_dependencies: from dumbmake.dumbmake import (dependency_map, add_extra_dependencies) depfile = os.path.join(self.topsrcdir, 'build', 'dumbmake-dependencies') with open(depfile) as f: dm = dependency_map(f.readlines()) new_pairs = list(add_extra_dependencies(target_pairs, dm)) self.log(logging.DEBUG, 'dumbmake', {'target_pairs': target_pairs, 'new_pairs': new_pairs}, 'Added extra dependencies: will build {new_pairs} ' + 'instead of {target_pairs}.') target_pairs = new_pairs # Ensure build backend is up to date. The alternative is to # have rules in the invoked Makefile to rebuild the build # backend. But that involves make reinvoking itself and there # are undesired side-effects of this. See bug 877308 for a # comprehensive history lesson. self._run_make(directory=self.topobjdir, target='backend.RecursiveMakeBackend', line_handler=output.on_line, log=False, print_directory=False) # Build target pairs. for make_dir, make_target in target_pairs: # We don't display build status messages during partial # tree builds because they aren't reliable there. This # could potentially be fixed if the build monitor were more # intelligent about encountering undefined state. status = self._run_make(directory=make_dir, target=make_target, line_handler=output.on_line, log=False, print_directory=False, ensure_exit_code=False, num_jobs=jobs, silent=not verbose, append_env={b'NO_BUILDSTATUS_MESSAGES': b'1'}) if status != 0: break else: monitor.start_resource_recording() status = self._run_make(srcdir=True, filename='client.mk', line_handler=output.on_line, log=False, print_directory=False, allow_parallel=False, ensure_exit_code=False, num_jobs=jobs, silent=not verbose) make_extra = self.mozconfig['make_extra'] or [] make_extra = dict(m.split('=', 1) for m in make_extra) # For universal builds, we need to run the automation steps in # the first architecture from MOZ_BUILD_PROJECTS projects = make_extra.get('MOZ_BUILD_PROJECTS') if projects: subdir = os.path.join(self.topobjdir, projects.split()[0]) else: subdir = self.topobjdir moz_automation = os.getenv('MOZ_AUTOMATION') or make_extra.get('export MOZ_AUTOMATION', None) if moz_automation and status == 0: status = self._run_make(target='automation/build', directory=subdir, line_handler=output.on_line, log=False, print_directory=False, ensure_exit_code=False, num_jobs=jobs, silent=not verbose) self.log(logging.WARNING, 'warning_summary', {'count': len(monitor.warnings_database)}, '{count} compiler warnings present.') monitor.finish(record_usage=status==0) high_finder, finder_percent = monitor.have_high_finder_usage() if high_finder: print(FINDER_SLOW_MESSAGE % finder_percent) ccache_end = monitor.ccache_stats() if ccache_start and ccache_end: ccache_diff = ccache_end - ccache_start if ccache_diff: self.log(logging.INFO, 'ccache', {'msg': ccache_diff.hit_rate_message()}, "{msg}") if monitor.elapsed > 300: # Display a notification when the build completes. self.notify('Build complete') if status: return status long_build = monitor.elapsed > 600 if long_build: output.on_line('We know it took a while, but your build finally finished successfully!') else: output.on_line('Your build was successful!') if monitor.have_resource_usage: excessive, swap_in, swap_out = monitor.have_excessive_swapping() # if excessive: # print(EXCESSIVE_SWAP_MESSAGE) print('To view resource usage of the build, run |mach ' 'resource-usage|.') # Only for full builds because incremental builders likely don't # need to be burdened with this. if not what: try: # Fennec doesn't have useful output from just building. We should # arguably make the build action useful for Fennec. Another day... if self.substs['MOZ_BUILD_APP'] != 'mobile/android': print('To take your build for a test drive, run: |mach run|') app = self.substs['MOZ_BUILD_APP'] if app in ('browser', 'mobile/android'): print('For more information on what to do now, see ' 'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox') except Exception: # Ignore Exceptions in case we can't find config.status (such # as when doing OSX Universal builds) pass return status
def build(self, what=None, pymake=False, disable_extra_make_dependencies=None, jobs=0, verbose=False): import which from mozbuild.controller.building import BuildMonitor from mozbuild.util import resolve_target_to_make self.log_manager.register_structured_logger(logging.getLogger('mozbuild')) warnings_path = self._get_state_filename('warnings.json') monitor = self._spawn(BuildMonitor) monitor.init(warnings_path) with BuildOutputManager(self.log_manager, monitor) as output: monitor.start() if what: top_make = os.path.join(self.topobjdir, 'Makefile') if not os.path.exists(top_make): print('Your tree has not been configured yet. Please run ' '|mach build| with no arguments.') return 1 # Collect target pairs. target_pairs = [] for target in what: path_arg = self._wrap_path_argument(target) make_dir, make_target = resolve_target_to_make(self.topobjdir, path_arg.relpath()) if make_dir is None and make_target is None: return 1 # See bug 886162 - we don't want to "accidentally" build # the entire tree (if that's really the intent, it's # unlikely they would have specified a directory.) if not make_dir and not make_target: print("The specified directory doesn't contain a " "Makefile and the first parent with one is the " "root of the tree. Please specify a directory " "with a Makefile or run |mach build| if you " "want to build the entire tree.") return 1 target_pairs.append((make_dir, make_target)) # Possibly add extra make depencies using dumbmake. if not disable_extra_make_dependencies: from dumbmake.dumbmake import (dependency_map, add_extra_dependencies) depfile = os.path.join(self.topsrcdir, 'build', 'dumbmake-dependencies') with open(depfile) as f: dm = dependency_map(f.readlines()) new_pairs = list(add_extra_dependencies(target_pairs, dm)) self.log(logging.DEBUG, 'dumbmake', {'target_pairs': target_pairs, 'new_pairs': new_pairs}, 'Added extra dependencies: will build {new_pairs} ' + 'instead of {target_pairs}.') target_pairs = new_pairs # Ensure build backend is up to date. The alternative is to # have rules in the invoked Makefile to rebuild the build # backend. But that involves make reinvoking itself and there # are undesired side-effects of this. See bug 877308 for a # comprehensive history lesson. self._run_make(directory=self.topobjdir, target='backend.RecursiveMakeBackend', force_pymake=pymake, line_handler=output.on_line, log=False, print_directory=False) # Build target pairs. for make_dir, make_target in target_pairs: # We don't display build status messages during partial # tree builds because they aren't reliable there. This # could potentially be fixed if the build monitor were more # intelligent about encountering undefined state. status = self._run_make(directory=make_dir, target=make_target, line_handler=output.on_line, log=False, print_directory=False, ensure_exit_code=False, num_jobs=jobs, silent=not verbose, append_env={b'NO_BUILDSTATUS_MESSAGES': b'1'}, force_pymake=pymake) if status != 0: break else: monitor.start_resource_recording() status = self._run_make(srcdir=True, filename='client.mk', line_handler=output.on_line, log=False, print_directory=False, allow_parallel=False, ensure_exit_code=False, num_jobs=jobs, silent=not verbose, force_pymake=pymake) self.log(logging.WARNING, 'warning_summary', {'count': len(monitor.warnings_database)}, '{count} compiler warnings present.') monitor.finish(record_usage=status==0) high_finder, finder_percent = monitor.have_high_finder_usage() if high_finder: print(FINDER_SLOW_MESSAGE % finder_percent) if monitor.elapsed > 300: # Display a notification when the build completes. # This could probably be uplifted into the mach core or at least # into a helper API. It is here as an experimentation to see how it # is received. try: if sys.platform.startswith('darwin'): notifier = which.which('terminal-notifier') self.run_process([notifier, '-title', 'Mozilla Build System', '-group', 'mozbuild', '-message', 'Build complete'], ensure_exit_code=False) except which.WhichError: pass except Exception as e: self.log(logging.WARNING, 'notifier-failed', {'error': e.message}, 'Notification center failed: {error}') if status: return status long_build = monitor.elapsed > 600 if long_build: print('We know it took a while, but your build finally finished successfully!') else: print('Your build was successful!') if monitor.have_resource_usage: excessive, swap_in, swap_out = monitor.have_excessive_swapping() # if excessive: # print(EXCESSIVE_SWAP_MESSAGE) print('To view resource usage of the build, run |mach ' 'resource-usage|.') # Only for full builds because incremental builders likely don't # need to be burdened with this. if not what: # Fennec doesn't have useful output from just building. We should # arguably make the build action useful for Fennec. Another day... if self.substs['MOZ_BUILD_APP'] != 'mobile/android': app_path = self.get_binary_path('app') print('To take your build for a test drive, run: %s' % app_path) app = self.substs['MOZ_BUILD_APP'] if app in ('browser', 'mobile/android'): print('For more information on what to do now, see ' 'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox') return status
def build(self, what=None, disable_extra_make_dependencies=None, jobs=0, directory=None, verbose=False, keep_going=False, mach_context=None): """Invoke the build backend. ``what`` defines the thing to build. If not defined, the default target is used. """ warnings_path = self._get_state_filename('warnings.json') monitor = self._spawn(BuildMonitor) monitor.init(warnings_path) ccache_start = monitor.ccache_stats() footer = BuildProgressFooter(self.log_manager.terminal, monitor) # Disable indexing in objdir because it is not necessary and can slow # down builds. mkdir(self.topobjdir, not_indexed=True) with BuildOutputManager(self.log_manager, monitor, footer) as output: monitor.start() if directory is not None and not what: print('Can only use -C/--directory with an explicit target ' 'name.') return 1 if directory is not None: disable_extra_make_dependencies = True directory = mozpath.normsep(directory) if directory.startswith('/'): directory = directory[1:] status = None monitor.start_resource_recording() if what: top_make = os.path.join(self.topobjdir, 'Makefile') if not os.path.exists(top_make): print('Your tree has not been configured yet. Please run ' '|mach build| with no arguments.') return 1 # Collect target pairs. target_pairs = [] for target in what: path_arg = self._wrap_path_argument(target) if directory is not None: make_dir = os.path.join(self.topobjdir, directory) make_target = target else: make_dir, make_target = \ resolve_target_to_make(self.topobjdir, path_arg.relpath()) if make_dir is None and make_target is None: return 1 # See bug 886162 - we don't want to "accidentally" build # the entire tree (if that's really the intent, it's # unlikely they would have specified a directory.) if not make_dir and not make_target: print("The specified directory doesn't contain a " "Makefile and the first parent with one is the " "root of the tree. Please specify a directory " "with a Makefile or run |mach build| if you " "want to build the entire tree.") return 1 target_pairs.append((make_dir, make_target)) # Possibly add extra make depencies using dumbmake. if not disable_extra_make_dependencies: from dumbmake.dumbmake import (dependency_map, add_extra_dependencies) depfile = os.path.join(self.topsrcdir, 'build', 'dumbmake-dependencies') with open(depfile) as f: dm = dependency_map(f.readlines()) new_pairs = list(add_extra_dependencies(target_pairs, dm)) self.log( logging.DEBUG, 'dumbmake', { 'target_pairs': target_pairs, 'new_pairs': new_pairs }, 'Added extra dependencies: will build {new_pairs} ' + 'instead of {target_pairs}.') target_pairs = new_pairs # Ensure build backend is up to date. The alternative is to # have rules in the invoked Makefile to rebuild the build # backend. But that involves make reinvoking itself and there # are undesired side-effects of this. See bug 877308 for a # comprehensive history lesson. self._run_make(directory=self.topobjdir, target='backend', line_handler=output.on_line, log=False, print_directory=False, keep_going=keep_going) # Build target pairs. for make_dir, make_target in target_pairs: # We don't display build status messages during partial # tree builds because they aren't reliable there. This # could potentially be fixed if the build monitor were more # intelligent about encountering undefined state. status = self._run_make( directory=make_dir, target=make_target, line_handler=output.on_line, log=False, print_directory=False, ensure_exit_code=False, num_jobs=jobs, silent=not verbose, append_env={b'NO_BUILDSTATUS_MESSAGES': b'1'}, keep_going=keep_going) if status != 0: break else: # Try to call the default backend's build() method. This will # run configure to determine BUILD_BACKENDS if it hasn't run # yet. config = None try: config = self.config_environment except Exception: config_rc = self.configure(buildstatus_messages=True, line_handler=output.on_line) if config_rc != 0: return config_rc # Even if configure runs successfully, we may have trouble # getting the config_environment for some builds, such as # OSX Universal builds. These have to go through client.mk # regardless. try: config = self.config_environment except Exception: pass if config: active_backend = config.substs.get('BUILD_BACKENDS', [None])[0] if active_backend: backend_cls = get_backend_class(active_backend)(config) status = backend_cls.build(self, output, jobs, verbose) # If the backend doesn't specify a build() method, then just # call client.mk directly. if status is None: status = self._run_make(srcdir=True, filename='client.mk', line_handler=output.on_line, log=False, print_directory=False, allow_parallel=False, ensure_exit_code=False, num_jobs=jobs, silent=not verbose, keep_going=keep_going) self.log(logging.WARNING, 'warning_summary', {'count': len(monitor.warnings_database)}, '{count} compiler warnings present.') monitor.finish(record_usage=status == 0) # Print the collected compiler warnings. This is redundant with # inline output from the compiler itself. However, unlike inline # output, this list is sorted and grouped by file, making it # easier to triage output. # # Only do this if we had a successful build. If the build failed, # there are more important things in the log to look for than # whatever code we warned about. if not status: # Suppress warnings for 3rd party projects in local builds # until we suppress them for real. # TODO remove entries/feature once we stop generating warnings # in these directories. pathToThirdparty = os.path.join(self.topsrcdir, "tools", "rewriting", "ThirdPartyPaths.txt") if os.path.exists(pathToThirdparty): with open(pathToThirdparty) as f: # Normalize the path (no trailing /) LOCAL_SUPPRESS_DIRS = tuple( d.rstrip('/') for d in f.read().splitlines()) else: # For application based on gecko like thunderbird LOCAL_SUPPRESS_DIRS = () suppressed_by_dir = Counter() for warning in sorted(monitor.instance_warnings): path = mozpath.normsep(warning['filename']) if path.startswith(self.topsrcdir): path = path[len(self.topsrcdir) + 1:] warning['normpath'] = path if (path.startswith(LOCAL_SUPPRESS_DIRS) and 'MOZ_AUTOMATION' not in os.environ): for d in LOCAL_SUPPRESS_DIRS: if path.startswith(d): suppressed_by_dir[d] += 1 break continue if warning['column'] is not None: self.log( logging.WARNING, 'compiler_warning', warning, 'warning: {normpath}:{line}:{column} [{flag}] ' '{message}') else: self.log(logging.WARNING, 'compiler_warning', warning, 'warning: {normpath}:{line} [{flag}] {message}') for d, count in sorted(suppressed_by_dir.items()): self.log(logging.WARNING, 'suppressed_warning', { 'dir': d, 'count': count }, '(suppressed {count} warnings in {dir})') high_finder, finder_percent = monitor.have_high_finder_usage() if high_finder: print(FINDER_SLOW_MESSAGE % finder_percent) ccache_end = monitor.ccache_stats() ccache_diff = None if ccache_start and ccache_end: ccache_diff = ccache_end - ccache_start if ccache_diff: self.log(logging.INFO, 'ccache', {'msg': ccache_diff.hit_rate_message()}, "{msg}") notify_minimum_time = 300 try: notify_minimum_time = int( os.environ.get('MACH_NOTIFY_MINTIME', '300')) except ValueError: # Just stick with the default pass if monitor.elapsed > notify_minimum_time: # Display a notification when the build completes. self.notify('Build complete' if not status else 'Build failed') if status: return status long_build = monitor.elapsed > 600 if long_build: output.on_line( 'We know it took a while, but your build finally finished successfully!' ) else: output.on_line('Your build was successful!') if monitor.have_resource_usage: excessive, swap_in, swap_out = monitor.have_excessive_swapping() # if excessive: # print(EXCESSIVE_SWAP_MESSAGE) print('To view resource usage of the build, run |mach ' 'resource-usage|.') telemetry_handler = getattr(mach_context, 'telemetry_handler', None) telemetry_data = monitor.get_resource_usage() # Record build configuration data. For now, we cherry pick # items we need rather than grabbing everything, in order # to avoid accidentally disclosing PII. telemetry_data['substs'] = {} try: for key in [ 'MOZ_ARTIFACT_BUILDS', 'MOZ_USING_CCACHE', 'MOZ_USING_SCCACHE' ]: value = self.substs.get(key, False) telemetry_data['substs'][key] = value except BuildEnvironmentNotFoundException: pass # Grab ccache stats if available. We need to be careful not # to capture information that can potentially identify the # user (such as the cache location) if ccache_diff: telemetry_data['ccache'] = {} for key in [key[0] for key in ccache_diff.STATS_KEYS]: try: telemetry_data['ccache'][key] = ccache_diff._values[ key] except KeyError: pass if telemetry_handler: telemetry_handler(mach_context, telemetry_data) # Only for full builds because incremental builders likely don't # need to be burdened with this. if not what: try: # Fennec doesn't have useful output from just building. We should # arguably make the build action useful for Fennec. Another day... if self.substs['MOZ_BUILD_APP'] != 'mobile/android': print( 'To take your build for a test drive, run: |mach run|') app = self.substs['MOZ_BUILD_APP'] if app in ('browser', 'mobile/android'): print( 'For more information on what to do now, see ' 'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox' ) except Exception: # Ignore Exceptions in case we can't find config.status (such # as when doing OSX Universal builds) pass return status
def build(self, what=None, pymake=False, disable_extra_make_dependencies=None, jobs=0, verbose=False): import which from mozbuild.controller.building import BuildMonitor from mozbuild.util import resolve_target_to_make self.log_manager.register_structured_logger( logging.getLogger('mozbuild')) warnings_path = self._get_state_filename('warnings.json') monitor = self._spawn(BuildMonitor) monitor.init(warnings_path) with BuildOutputManager(self.log_manager, monitor) as output: monitor.start() if what: top_make = os.path.join(self.topobjdir, 'Makefile') if not os.path.exists(top_make): print('Your tree has not been configured yet. Please run ' '|mach build| with no arguments.') return 1 # Collect target pairs. target_pairs = [] for target in what: path_arg = self._wrap_path_argument(target) make_dir, make_target = resolve_target_to_make( self.topobjdir, path_arg.relpath()) if make_dir is None and make_target is None: return 1 # See bug 886162 - we don't want to "accidentally" build # the entire tree (if that's really the intent, it's # unlikely they would have specified a directory.) if not make_dir and not make_target: print("The specified directory doesn't contain a " "Makefile and the first parent with one is the " "root of the tree. Please specify a directory " "with a Makefile or run |mach build| if you " "want to build the entire tree.") return 1 target_pairs.append((make_dir, make_target)) # Possibly add extra make depencies using dumbmake. if not disable_extra_make_dependencies: from dumbmake.dumbmake import (dependency_map, add_extra_dependencies) depfile = os.path.join(self.topsrcdir, 'build', 'dumbmake-dependencies') with open(depfile) as f: dm = dependency_map(f.readlines()) new_pairs = list(add_extra_dependencies(target_pairs, dm)) self.log( logging.DEBUG, 'dumbmake', { 'target_pairs': target_pairs, 'new_pairs': new_pairs }, 'Added extra dependencies: will build {new_pairs} ' + 'instead of {target_pairs}.') target_pairs = new_pairs # Ensure build backend is up to date. The alternative is to # have rules in the invoked Makefile to rebuild the build # backend. But that involves make reinvoking itself and there # are undesired side-effects of this. See bug 877308 for a # comprehensive history lesson. self._run_make(directory=self.topobjdir, target='backend.RecursiveMakeBackend', force_pymake=pymake, line_handler=output.on_line, log=False, print_directory=False) # Build target pairs. for make_dir, make_target in target_pairs: # We don't display build status messages during partial # tree builds because they aren't reliable there. This # could potentially be fixed if the build monitor were more # intelligent about encountering undefined state. status = self._run_make( directory=make_dir, target=make_target, line_handler=output.on_line, log=False, print_directory=False, ensure_exit_code=False, num_jobs=jobs, silent=not verbose, append_env={b'NO_BUILDSTATUS_MESSAGES': b'1'}, force_pymake=pymake) if status != 0: break else: monitor.start_resource_recording() status = self._run_make(srcdir=True, filename='client.mk', line_handler=output.on_line, log=False, print_directory=False, allow_parallel=False, ensure_exit_code=False, num_jobs=jobs, silent=not verbose, force_pymake=pymake) self.log(logging.WARNING, 'warning_summary', {'count': len(monitor.warnings_database)}, '{count} compiler warnings present.') monitor.finish(record_usage=status == 0) high_finder, finder_percent = monitor.have_high_finder_usage() if high_finder: print(FINDER_SLOW_MESSAGE % finder_percent) if monitor.elapsed > 300: # Display a notification when the build completes. # This could probably be uplifted into the mach core or at least # into a helper API. It is here as an experimentation to see how it # is received. try: if sys.platform.startswith('darwin'): notifier = which.which('terminal-notifier') self.run_process([ notifier, '-title', 'Mozilla Build System', '-group', 'mozbuild', '-message', 'Build complete' ], ensure_exit_code=False) except which.WhichError: pass except Exception as e: self.log(logging.WARNING, 'notifier-failed', {'error': e.message}, 'Notification center failed: {error}') if status: return status long_build = monitor.elapsed > 600 if long_build: print( 'We know it took a while, but your build finally finished successfully!' ) else: print('Your build was successful!') if monitor.have_resource_usage: excessive, swap_in, swap_out = monitor.have_excessive_swapping() # if excessive: # print(EXCESSIVE_SWAP_MESSAGE) print('To view resource usage of the build, run |mach ' 'resource-usage|.') # Only for full builds because incremental builders likely don't # need to be burdened with this. if not what: # Fennec doesn't have useful output from just building. We should # arguably make the build action useful for Fennec. Another day... if self.substs['MOZ_BUILD_APP'] != 'mobile/android': print('To take your build for a test drive, run: |mach run|') app = self.substs['MOZ_BUILD_APP'] if app in ('browser', 'mobile/android'): print( 'For more information on what to do now, see ' 'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox' ) return status
def build(self, what=None, disable_extra_make_dependencies=None, jobs=0, directory=None, verbose=False): import which from mozbuild.controller.building import BuildMonitor from mozbuild.util import resolve_target_to_make self.log_manager.register_structured_logger(logging.getLogger('mozbuild')) warnings_path = self._get_state_filename('warnings.json') monitor = self._spawn(BuildMonitor) monitor.init(warnings_path) ccache_start = monitor.ccache_stats() with BuildOutputManager(self.log_manager, monitor) as output: monitor.start() if directory is not None and not what: print('Can only use -C/--directory with an explicit target ' 'name.') return 1 if directory is not None: disable_extra_make_dependencies=True directory = mozpath.normsep(directory) if directory.startswith('/'): directory = directory[1:] if what: top_make = os.path.join(self.topobjdir, 'Makefile') if not os.path.exists(top_make): print('Your tree has not been configured yet. Please run ' '|mach build| with no arguments.') return 1 # Collect target pairs. target_pairs = [] for target in what: path_arg = self._wrap_path_argument(target) if directory is not None: make_dir = os.path.join(self.topobjdir, directory) make_target = target else: make_dir, make_target = \ resolve_target_to_make(self.topobjdir, path_arg.relpath()) if make_dir is None and make_target is None: return 1 # See bug 886162 - we don't want to "accidentally" build # the entire tree (if that's really the intent, it's # unlikely they would have specified a directory.) if not make_dir and not make_target: print("The specified directory doesn't contain a " "Makefile and the first parent with one is the " "root of the tree. Please specify a directory " "with a Makefile or run |mach build| if you " "want to build the entire tree.") return 1 target_pairs.append((make_dir, make_target)) # Possibly add extra make depencies using dumbmake. if not disable_extra_make_dependencies: from dumbmake.dumbmake import (dependency_map, add_extra_dependencies) depfile = os.path.join(self.topsrcdir, 'build', 'dumbmake-dependencies') with open(depfile) as f: dm = dependency_map(f.readlines()) new_pairs = list(add_extra_dependencies(target_pairs, dm)) self.log(logging.DEBUG, 'dumbmake', {'target_pairs': target_pairs, 'new_pairs': new_pairs}, 'Added extra dependencies: will build {new_pairs} ' + 'instead of {target_pairs}.') target_pairs = new_pairs # Ensure build backend is up to date. The alternative is to # have rules in the invoked Makefile to rebuild the build # backend. But that involves make reinvoking itself and there # are undesired side-effects of this. See bug 877308 for a # comprehensive history lesson. self._run_make(directory=self.topobjdir, target='backend.RecursiveMakeBackend', line_handler=output.on_line, log=False, print_directory=False) # Build target pairs. for make_dir, make_target in target_pairs: # We don't display build status messages during partial # tree builds because they aren't reliable there. This # could potentially be fixed if the build monitor were more # intelligent about encountering undefined state. status = self._run_make(directory=make_dir, target=make_target, line_handler=output.on_line, log=False, print_directory=False, ensure_exit_code=False, num_jobs=jobs, silent=not verbose, append_env={b'NO_BUILDSTATUS_MESSAGES': b'1'}) if status != 0: break else: monitor.start_resource_recording() status = self._run_make(srcdir=True, filename='client.mk', line_handler=output.on_line, log=False, print_directory=False, allow_parallel=False, ensure_exit_code=False, num_jobs=jobs, silent=not verbose) make_extra = self.mozconfig['make_extra'] or [] make_extra = dict(m.split('=', 1) for m in make_extra) # For universal builds, we need to run the automation steps in # the first architecture from MOZ_BUILD_PROJECTS projects = make_extra.get('MOZ_BUILD_PROJECTS') if projects: subdir = os.path.join(self.topobjdir, projects.split()[0]) else: subdir = self.topobjdir moz_automation = os.getenv('MOZ_AUTOMATION') or make_extra.get('export MOZ_AUTOMATION', None) if moz_automation and status == 0: status = self._run_make(target='automation/build', directory=subdir, line_handler=output.on_line, log=False, print_directory=False, ensure_exit_code=False, num_jobs=jobs, silent=not verbose) self.log(logging.WARNING, 'warning_summary', {'count': len(monitor.warnings_database)}, '{count} compiler warnings present.') monitor.finish(record_usage=status==0) high_finder, finder_percent = monitor.have_high_finder_usage() if high_finder: print(FINDER_SLOW_MESSAGE % finder_percent) ccache_end = monitor.ccache_stats() if ccache_start and ccache_end: ccache_diff = ccache_end - ccache_start if ccache_diff: self.log(logging.INFO, 'ccache', {'msg': ccache_diff.hit_rate_message()}, "{msg}") moz_nospam = os.environ.get('MOZ_NOSPAM') if monitor.elapsed > 300 and not moz_nospam: # Display a notification when the build completes. # This could probably be uplifted into the mach core or at least # into a helper API. It is here as an experimentation to see how it # is received. try: if sys.platform.startswith('darwin'): try: notifier = which.which('terminal-notifier') except which.WhichError: raise Exception('Install terminal-notifier to get ' 'a notification when the build finishes.') self.run_process([notifier, '-title', 'Mozilla Build System', '-group', 'mozbuild', '-message', 'Build complete'], ensure_exit_code=False) elif sys.platform.startswith('linux'): try: import dbus except ImportError: raise Exception('Install the python dbus module to ' 'get a notification when the build finishes.') bus = dbus.SessionBus() notify = bus.get_object('org.freedesktop.Notifications', '/org/freedesktop/Notifications') method = notify.get_dbus_method('Notify', 'org.freedesktop.Notifications') method('Mozilla Build System', 0, '', 'Build complete', '', [], [], -1) elif sys.platform.startswith('win'): from ctypes import Structure, windll, POINTER, sizeof from ctypes.wintypes import DWORD, HANDLE, WINFUNCTYPE, BOOL, UINT class FLASHWINDOW(Structure): _fields_ = [("cbSize", UINT), ("hwnd", HANDLE), ("dwFlags", DWORD), ("uCount", UINT), ("dwTimeout", DWORD)] FlashWindowExProto = WINFUNCTYPE(BOOL, POINTER(FLASHWINDOW)) FlashWindowEx = FlashWindowExProto(("FlashWindowEx", windll.user32)) FLASHW_CAPTION = 0x01 FLASHW_TRAY = 0x02 FLASHW_TIMERNOFG = 0x0C params = FLASHWINDOW(sizeof(FLASHWINDOW), windll.kernel32.GetConsoleWindow(), FLASHW_CAPTION | FLASHW_TRAY | FLASHW_TIMERNOFG, 3, 0) FlashWindowEx(params) except Exception as e: self.log(logging.WARNING, 'notifier-failed', {'error': e.message}, 'Notification center failed: {error}') if status: return status long_build = monitor.elapsed > 600 if long_build: output.on_line('We know it took a while, but your build finally finished successfully!') else: output.on_line('Your build was successful!') if monitor.have_resource_usage: excessive, swap_in, swap_out = monitor.have_excessive_swapping() # if excessive: # print(EXCESSIVE_SWAP_MESSAGE) print('To view resource usage of the build, run |mach ' 'resource-usage|.') # Only for full builds because incremental builders likely don't # need to be burdened with this. if not what: try: # Fennec doesn't have useful output from just building. We should # arguably make the build action useful for Fennec. Another day... if self.substs['MOZ_BUILD_APP'] != 'mobile/android': print('To take your build for a test drive, run: |mach run|') app = self.substs['MOZ_BUILD_APP'] if app in ('browser', 'mobile/android'): print('For more information on what to do now, see ' 'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox') except Exception: # Ignore Exceptions in case we can't find config.status (such # as when doing OSX Universal builds) pass return status
def build(self, what=None, disable_extra_make_dependencies=None, jobs=0, verbose=False): import which from mozbuild.controller.building import BuildMonitor from mozbuild.util import resolve_target_to_make self.log_manager.register_structured_logger( logging.getLogger('mozbuild')) warnings_path = self._get_state_filename('warnings.json') monitor = self._spawn(BuildMonitor) monitor.init(warnings_path) ccache_start = monitor.ccache_stats() with BuildOutputManager(self.log_manager, monitor) as output: monitor.start() if what: top_make = os.path.join(self.topobjdir, 'Makefile') if not os.path.exists(top_make): print('Your tree has not been configured yet. Please run ' '|mach build| with no arguments.') return 1 # Collect target pairs. target_pairs = [] for target in what: path_arg = self._wrap_path_argument(target) make_dir, make_target = resolve_target_to_make( self.topobjdir, path_arg.relpath()) if make_dir is None and make_target is None: return 1 # See bug 886162 - we don't want to "accidentally" build # the entire tree (if that's really the intent, it's # unlikely they would have specified a directory.) if not make_dir and not make_target: print("The specified directory doesn't contain a " "Makefile and the first parent with one is the " "root of the tree. Please specify a directory " "with a Makefile or run |mach build| if you " "want to build the entire tree.") return 1 target_pairs.append((make_dir, make_target)) # Possibly add extra make depencies using dumbmake. if not disable_extra_make_dependencies: from dumbmake.dumbmake import (dependency_map, add_extra_dependencies) depfile = os.path.join(self.topsrcdir, 'build', 'dumbmake-dependencies') with open(depfile) as f: dm = dependency_map(f.readlines()) new_pairs = list(add_extra_dependencies(target_pairs, dm)) self.log( logging.DEBUG, 'dumbmake', { 'target_pairs': target_pairs, 'new_pairs': new_pairs }, 'Added extra dependencies: will build {new_pairs} ' + 'instead of {target_pairs}.') target_pairs = new_pairs # Ensure build backend is up to date. The alternative is to # have rules in the invoked Makefile to rebuild the build # backend. But that involves make reinvoking itself and there # are undesired side-effects of this. See bug 877308 for a # comprehensive history lesson. self._run_make(directory=self.topobjdir, target='backend.RecursiveMakeBackend', line_handler=output.on_line, log=False, print_directory=False) # Build target pairs. for make_dir, make_target in target_pairs: # We don't display build status messages during partial # tree builds because they aren't reliable there. This # could potentially be fixed if the build monitor were more # intelligent about encountering undefined state. status = self._run_make( directory=make_dir, target=make_target, line_handler=output.on_line, log=False, print_directory=False, ensure_exit_code=False, num_jobs=jobs, silent=not verbose, append_env={b'NO_BUILDSTATUS_MESSAGES': b'1'}) if status != 0: break else: monitor.start_resource_recording() status = self._run_make(srcdir=True, filename='client.mk', line_handler=output.on_line, log=False, print_directory=False, allow_parallel=False, ensure_exit_code=False, num_jobs=jobs, silent=not verbose) make_extra = self.mozconfig['make_extra'] or [] make_extra = dict(m.split('=', 1) for m in make_extra) # For universal builds, we need to run the automation steps in # the first architecture from MOZ_BUILD_PROJECTS projects = make_extra.get('MOZ_BUILD_PROJECTS') if projects: subdir = os.path.join(self.topobjdir, projects.split()[0]) else: subdir = self.topobjdir moz_automation = os.getenv('MOZ_AUTOMATION') or make_extra.get( 'export MOZ_AUTOMATION', None) if moz_automation and status == 0: status = self._run_make(target='automation/build', directory=subdir, line_handler=output.on_line, log=False, print_directory=False, ensure_exit_code=False, num_jobs=jobs, silent=not verbose) self.log(logging.WARNING, 'warning_summary', {'count': len(monitor.warnings_database)}, '{count} compiler warnings present.') monitor.finish(record_usage=status == 0) high_finder, finder_percent = monitor.have_high_finder_usage() if high_finder: print(FINDER_SLOW_MESSAGE % finder_percent) ccache_end = monitor.ccache_stats() if ccache_start and ccache_end: ccache_diff = ccache_end - ccache_start if ccache_diff: self.log(logging.INFO, 'ccache', {'msg': ccache_diff.hit_rate_message()}, "{msg}") moz_nospam = os.environ.get('MOZ_NOSPAM') if monitor.elapsed > 300 and not moz_nospam: # Display a notification when the build completes. # This could probably be uplifted into the mach core or at least # into a helper API. It is here as an experimentation to see how it # is received. try: if sys.platform.startswith('darwin'): try: notifier = which.which('terminal-notifier') except which.WhichError: raise Exception( 'Install terminal-notifier to get ' 'a notification when the build finishes.') self.run_process([ notifier, '-title', 'Mozilla Build System', '-group', 'mozbuild', '-message', 'Build complete' ], ensure_exit_code=False) elif sys.platform.startswith('linux'): try: import dbus except ImportError: raise Exception( 'Install the python dbus module to ' 'get a notification when the build finishes.') bus = dbus.SessionBus() notify = bus.get_object('org.freedesktop.Notifications', '/org/freedesktop/Notifications') method = notify.get_dbus_method( 'Notify', 'org.freedesktop.Notifications') method('Mozilla Build System', 0, '', 'Build complete', '', [], [], -1) elif sys.platform.startswith('win'): from ctypes import Structure, windll, POINTER, sizeof from ctypes.wintypes import DWORD, HANDLE, WINFUNCTYPE, BOOL, UINT class FLASHWINDOW(Structure): _fields_ = [("cbSize", UINT), ("hwnd", HANDLE), ("dwFlags", DWORD), ("uCount", UINT), ("dwTimeout", DWORD)] FlashWindowExProto = WINFUNCTYPE(BOOL, POINTER(FLASHWINDOW)) FlashWindowEx = FlashWindowExProto( ("FlashWindowEx", windll.user32)) FLASHW_CAPTION = 0x01 FLASHW_TRAY = 0x02 FLASHW_TIMERNOFG = 0x0C params = FLASHWINDOW( sizeof(FLASHWINDOW), windll.kernel32.GetConsoleWindow(), FLASHW_CAPTION | FLASHW_TRAY | FLASHW_TIMERNOFG, 3, 0) FlashWindowEx(params) except Exception as e: self.log(logging.WARNING, 'notifier-failed', {'error': e.message}, 'Notification center failed: {error}') if status: return status long_build = monitor.elapsed > 600 if long_build: output.on_line( 'We know it took a while, but your build finally finished successfully!' ) else: output.on_line('Your build was successful!') if monitor.have_resource_usage: excessive, swap_in, swap_out = monitor.have_excessive_swapping() # if excessive: # print(EXCESSIVE_SWAP_MESSAGE) print('To view resource usage of the build, run |mach ' 'resource-usage|.') # Only for full builds because incremental builders likely don't # need to be burdened with this. if not what: try: # Fennec doesn't have useful output from just building. We should # arguably make the build action useful for Fennec. Another day... if self.substs['MOZ_BUILD_APP'] != 'mobile/android': print( 'To take your build for a test drive, run: |mach run|') app = self.substs['MOZ_BUILD_APP'] if app in ('browser', 'mobile/android'): print( 'For more information on what to do now, see ' 'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox' ) except Exception: # Ignore Exceptions in case we can't find config.status (such # as when doing OSX Universal builds) pass return status
def build(self, what=None, disable_extra_make_dependencies=None, jobs=0, verbose=False): from mozbuild.controller.building import BuildMonitor from mozbuild.util import resolve_target_to_make warnings_path = self._get_state_filename('warnings.json') monitor = BuildMonitor(self.topobjdir, warnings_path) with BuildOutputManager(self.log_manager, monitor) as output: monitor.start() if what: top_make = os.path.join(self.topobjdir, 'Makefile') if not os.path.exists(top_make): print('Your tree has not been configured yet. Please run ' '|mach build| with no arguments.') return 1 # Collect target pairs. target_pairs = [] for target in what: path_arg = self._wrap_path_argument(target) make_dir, make_target = resolve_target_to_make( self.topobjdir, path_arg.relpath()) if make_dir is None and make_target is None: return 1 target_pairs.append((make_dir, make_target)) # Possibly add extra make depencies using dumbmake. if not disable_extra_make_dependencies: from dumbmake.dumbmake import (dependency_map, add_extra_dependencies) depfile = os.path.join(self.topsrcdir, 'build', 'dumbmake-dependencies') with open(depfile) as f: dm = dependency_map(f.readlines()) new_pairs = list(add_extra_dependencies(target_pairs, dm)) self.log( logging.DEBUG, 'dumbmake', { 'target_pairs': target_pairs, 'new_pairs': new_pairs }, 'Added extra dependencies: will build {new_pairs} ' + 'instead of {target_pairs}.') target_pairs = new_pairs # Build target pairs. for make_dir, make_target in target_pairs: status = self._run_make(directory=make_dir, target=make_target, line_handler=output.on_line, log=False, print_directory=False, ensure_exit_code=False, num_jobs=jobs, silent=not verbose) if status != 0: break else: status = self._run_make(srcdir=True, filename='client.mk', line_handler=output.on_line, log=False, print_directory=False, allow_parallel=False, ensure_exit_code=False, num_jobs=jobs, silent=not verbose) self.log(logging.WARNING, 'warning_summary', {'count': len(monitor.warnings_database)}, '{count} compiler warnings present.') monitor.finish() high_finder, finder_percent = monitor.have_high_finder_usage() if high_finder: print(FINDER_SLOW_MESSAGE % finder_percent) long_build = monitor.elapsed > 600 if status: return status if long_build: print( 'We know it took a while, but your build finally finished successfully!' ) else: print('Your build was successful!') # Only for full builds because incremental builders likely don't # need to be burdened with this. if not what: # Fennec doesn't have useful output from just building. We should # arguably make the build action useful for Fennec. Another day... if self.substs['MOZ_BUILD_APP'] != 'mobile/android': app_path = self.get_binary_path('app') print('To take your build for a test drive, run: %s' % app_path) app = self.substs['MOZ_BUILD_APP'] if app in ('browser', 'mobile/android'): print( 'Please remember that you also need to PACKAGE your build ' 'to have all components properly included and unnecessary files removed.' ) return status
def build(self, what=None, disable_extra_make_dependencies=None, jobs=0, directory=None, verbose=False, keep_going=False, mach_context=None): """Invoke the build backend. ``what`` defines the thing to build. If not defined, the default target is used. """ warnings_path = self._get_state_filename('warnings.json') monitor = self._spawn(BuildMonitor) monitor.init(warnings_path) ccache_start = monitor.ccache_stats() footer = BuildProgressFooter(self.log_manager.terminal, monitor) # Disable indexing in objdir because it is not necessary and can slow # down builds. mkdir(self.topobjdir, not_indexed=True) with BuildOutputManager(self.log_manager, monitor, footer) as output: monitor.start() if directory is not None and not what: print('Can only use -C/--directory with an explicit target ' 'name.') return 1 if directory is not None: disable_extra_make_dependencies=True directory = mozpath.normsep(directory) if directory.startswith('/'): directory = directory[1:] status = None monitor.start_resource_recording() if what: top_make = os.path.join(self.topobjdir, 'Makefile') if not os.path.exists(top_make): print('Your tree has not been configured yet. Please run ' '|mach build| with no arguments.') return 1 # Collect target pairs. target_pairs = [] for target in what: path_arg = self._wrap_path_argument(target) if directory is not None: make_dir = os.path.join(self.topobjdir, directory) make_target = target else: make_dir, make_target = \ resolve_target_to_make(self.topobjdir, path_arg.relpath()) if make_dir is None and make_target is None: return 1 # See bug 886162 - we don't want to "accidentally" build # the entire tree (if that's really the intent, it's # unlikely they would have specified a directory.) if not make_dir and not make_target: print("The specified directory doesn't contain a " "Makefile and the first parent with one is the " "root of the tree. Please specify a directory " "with a Makefile or run |mach build| if you " "want to build the entire tree.") return 1 target_pairs.append((make_dir, make_target)) # Possibly add extra make depencies using dumbmake. if not disable_extra_make_dependencies: from dumbmake.dumbmake import (dependency_map, add_extra_dependencies) depfile = os.path.join(self.topsrcdir, 'build', 'dumbmake-dependencies') with open(depfile) as f: dm = dependency_map(f.readlines()) new_pairs = list(add_extra_dependencies(target_pairs, dm)) self.log(logging.DEBUG, 'dumbmake', {'target_pairs': target_pairs, 'new_pairs': new_pairs}, 'Added extra dependencies: will build {new_pairs} ' + 'instead of {target_pairs}.') target_pairs = new_pairs # Ensure build backend is up to date. The alternative is to # have rules in the invoked Makefile to rebuild the build # backend. But that involves make reinvoking itself and there # are undesired side-effects of this. See bug 877308 for a # comprehensive history lesson. self._run_make(directory=self.topobjdir, target='backend', line_handler=output.on_line, log=False, print_directory=False, keep_going=keep_going) # Build target pairs. for make_dir, make_target in target_pairs: # We don't display build status messages during partial # tree builds because they aren't reliable there. This # could potentially be fixed if the build monitor were more # intelligent about encountering undefined state. status = self._run_make(directory=make_dir, target=make_target, line_handler=output.on_line, log=False, print_directory=False, ensure_exit_code=False, num_jobs=jobs, silent=not verbose, append_env={b'NO_BUILDSTATUS_MESSAGES': b'1'}, keep_going=keep_going) if status != 0: break else: # Try to call the default backend's build() method. This will # run configure to determine BUILD_BACKENDS if it hasn't run # yet. config = None try: config = self.config_environment except Exception: config_rc = self.configure(buildstatus_messages=True, line_handler=output.on_line) if config_rc != 0: return config_rc # Even if configure runs successfully, we may have trouble # getting the config_environment for some builds, such as # OSX Universal builds. These have to go through client.mk # regardless. try: config = self.config_environment except Exception: pass if config: active_backend = config.substs.get('BUILD_BACKENDS', [None])[0] if active_backend: backend_cls = get_backend_class(active_backend)(config) status = backend_cls.build(self, output, jobs, verbose) # If the backend doesn't specify a build() method, then just # call client.mk directly. if status is None: status = self._run_client_mk(line_handler=output.on_line, jobs=jobs, verbose=verbose, keep_going=keep_going) self.log(logging.WARNING, 'warning_summary', {'count': len(monitor.warnings_database)}, '{count} compiler warnings present.') # Try to run the active build backend's post-build step, if possible. try: config = self.config_environment active_backend = config.substs.get('BUILD_BACKENDS', [None])[0] if active_backend: backend_cls = get_backend_class(active_backend)(config) new_status = backend_cls.post_build(self, output, jobs, verbose, status) status = new_status except Exception as ex: self.log(logging.DEBUG, 'post_build', {'ex': ex}, "Unable to run active build backend's post-build step; " + "failing the build due to exception: {ex}.") if not status: # If the underlying build provided a failing status, pass # it through; otherwise, fail. status = 1 monitor.finish(record_usage=status == 0) # Print the collected compiler warnings. This is redundant with # inline output from the compiler itself. However, unlike inline # output, this list is sorted and grouped by file, making it # easier to triage output. # # Only do this if we had a successful build. If the build failed, # there are more important things in the log to look for than # whatever code we warned about. if not status: # Suppress warnings for 3rd party projects in local builds # until we suppress them for real. # TODO remove entries/feature once we stop generating warnings # in these directories. pathToThirdparty = os.path.join(self.topsrcdir, "tools", "rewriting", "ThirdPartyPaths.txt") if os.path.exists(pathToThirdparty): with open(pathToThirdparty) as f: # Normalize the path (no trailing /) LOCAL_SUPPRESS_DIRS = tuple(d.rstrip('/') for d in f.read().splitlines()) else: # For application based on gecko like thunderbird LOCAL_SUPPRESS_DIRS = () suppressed_by_dir = Counter() for warning in sorted(monitor.instance_warnings): path = mozpath.normsep(warning['filename']) if path.startswith(self.topsrcdir): path = path[len(self.topsrcdir) + 1:] warning['normpath'] = path if (path.startswith(LOCAL_SUPPRESS_DIRS) and 'MOZ_AUTOMATION' not in os.environ): for d in LOCAL_SUPPRESS_DIRS: if path.startswith(d): suppressed_by_dir[d] += 1 break continue if warning['column'] is not None: self.log(logging.WARNING, 'compiler_warning', warning, 'warning: {normpath}:{line}:{column} [{flag}] ' '{message}') else: self.log(logging.WARNING, 'compiler_warning', warning, 'warning: {normpath}:{line} [{flag}] {message}') for d, count in sorted(suppressed_by_dir.items()): self.log(logging.WARNING, 'suppressed_warning', {'dir': d, 'count': count}, '(suppressed {count} warnings in {dir})') high_finder, finder_percent = monitor.have_high_finder_usage() if high_finder: print(FINDER_SLOW_MESSAGE % finder_percent) ccache_end = monitor.ccache_stats() ccache_diff = None if ccache_start and ccache_end: ccache_diff = ccache_end - ccache_start if ccache_diff: self.log(logging.INFO, 'ccache', {'msg': ccache_diff.hit_rate_message()}, "{msg}") notify_minimum_time = 300 try: notify_minimum_time = int(os.environ.get('MACH_NOTIFY_MINTIME', '300')) except ValueError: # Just stick with the default pass if monitor.elapsed > notify_minimum_time: # Display a notification when the build completes. self.notify('Build complete' if not status else 'Build failed') if status: return status long_build = monitor.elapsed > 600 if long_build: output.on_line('We know it took a while, but your build finally finished successfully!') else: output.on_line('Your build was successful!') if monitor.have_resource_usage: excessive, swap_in, swap_out = monitor.have_excessive_swapping() # if excessive: # print(EXCESSIVE_SWAP_MESSAGE) print('To view resource usage of the build, run |mach ' 'resource-usage|.') telemetry_handler = getattr(mach_context, 'telemetry_handler', None) telemetry_data = monitor.get_resource_usage() # Record build configuration data. For now, we cherry pick # items we need rather than grabbing everything, in order # to avoid accidentally disclosing PII. telemetry_data['substs'] = {} try: for key in ['MOZ_ARTIFACT_BUILDS', 'MOZ_USING_CCACHE', 'MOZ_USING_SCCACHE']: value = self.substs.get(key, False) telemetry_data['substs'][key] = value except BuildEnvironmentNotFoundException: pass # Grab ccache stats if available. We need to be careful not # to capture information that can potentially identify the # user (such as the cache location) if ccache_diff: telemetry_data['ccache'] = {} for key in [key[0] for key in ccache_diff.STATS_KEYS]: try: telemetry_data['ccache'][key] = ccache_diff._values[key] except KeyError: pass if telemetry_handler: telemetry_handler(mach_context, telemetry_data) # Only for full builds because incremental builders likely don't # need to be burdened with this. if not what: try: # Fennec doesn't have useful output from just building. We should # arguably make the build action useful for Fennec. Another day... if self.substs['MOZ_BUILD_APP'] != 'mobile/android': print('To take your build for a test drive, run: |mach run|') app = self.substs['MOZ_BUILD_APP'] if app in ('browser', 'mobile/android'): print('For more information on what to do now, see ' 'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox') except Exception: # Ignore Exceptions in case we can't find config.status (such # as when doing OSX Universal builds) pass return status