def build(self, what=None, disable_extra_make_dependencies=None, jobs=0, verbose=False):
from mozbuild.controller.building import BuildMonitor
from mozbuild.util import resolve_target_to_make
warnings_path = self._get_state_filename('warnings.json')
monitor = BuildMonitor(self.topobjdir, warnings_path)
with BuildOutputManager(self.log_manager, monitor) as output:
monitor.start()
if what:
top_make = os.path.join(self.topobjdir, 'Makefile')
if not os.path.exists(top_make):
print('Your tree has not been configured yet. Please run '
'|mach build| with no arguments.')
return 1
# Collect target pairs.
target_pairs = []
for target in what:
path_arg = self._wrap_path_argument(target)
make_dir, make_target = resolve_target_to_make(self.topobjdir,
path_arg.relpath())
if make_dir is None and make_target is None:
return 1
target_pairs.append((make_dir, make_target))
# Possibly add extra make depencies using dumbmake.
if not disable_extra_make_dependencies:
from dumbmake.dumbmake import (dependency_map,
add_extra_dependencies)
depfile = os.path.join(self.topsrcdir, 'build',
'dumbmake-dependencies')
with open(depfile) as f:
dm = dependency_map(f.readlines())
new_pairs = list(add_extra_dependencies(target_pairs, dm))
self.log(logging.DEBUG, 'dumbmake',
{'target_pairs': target_pairs,
'new_pairs': new_pairs},
'Added extra dependencies: will build {new_pairs} ' +
'instead of {target_pairs}.')
target_pairs = new_pairs
# Build target pairs.
for make_dir, make_target in target_pairs:
status = self._run_make(directory=make_dir, target=make_target,
line_handler=output.on_line, log=False, print_directory=False,
ensure_exit_code=False, num_jobs=jobs, silent=not verbose)
if status != 0:
break
else:
status = self._run_make(srcdir=True, filename='client.mk',
line_handler=output.on_line, log=False, print_directory=False,
allow_parallel=False, ensure_exit_code=False, num_jobs=jobs,
silent=not verbose)
self.log(logging.WARNING, 'warning_summary',
{'count': len(monitor.warnings_database)},
'{count} compiler warnings present.')
monitor.finish()
high_finder, finder_percent = monitor.have_high_finder_usage()
if high_finder:
print(FINDER_SLOW_MESSAGE % finder_percent)
long_build = monitor.elapsed > 600
if status:
return status
if long_build:
print('We know it took a while, but your build finally finished successfully!')
else:
print('Your build was successful!')
# Only for full builds because incremental builders likely don't
# need to be burdened with this.
if not what:
# Fennec doesn't have useful output from just building. We should
# arguably make the build action useful for Fennec. Another day...
if self.substs['MOZ_BUILD_APP'] != 'mobile/android':
app_path = self.get_binary_path('app')
print('To take your build for a test drive, run: %s' % app_path)
app = self.substs['MOZ_BUILD_APP']
if app in ('browser', 'mobile/android'):
print('For more information on what to do now, see '
'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox')
return status
def build(self, what=None, disable_extra_make_dependencies=None, jobs=0,
directory=None, verbose=False):
import which
from mozbuild.controller.building import BuildMonitor
from mozbuild.util import resolve_target_to_make
self.log_manager.register_structured_logger(logging.getLogger('mozbuild'))
warnings_path = self._get_state_filename('warnings.json')
monitor = self._spawn(BuildMonitor)
monitor.init(warnings_path)
ccache_start = monitor.ccache_stats()
with BuildOutputManager(self.log_manager, monitor) as output:
monitor.start()
if directory is not None and not what:
print('Can only use -C/--directory with an explicit target '
'name.')
return 1
if directory is not None:
disable_extra_make_dependencies=True
directory = mozpath.normsep(directory)
if directory.startswith('/'):
directory = directory[1:]
if what:
top_make = os.path.join(self.topobjdir, 'Makefile')
if not os.path.exists(top_make):
print('Your tree has not been configured yet. Please run '
'|mach build| with no arguments.')
return 1
# Collect target pairs.
target_pairs = []
for target in what:
path_arg = self._wrap_path_argument(target)
if directory is not None:
make_dir = os.path.join(self.topobjdir, directory)
make_target = target
else:
make_dir, make_target = \
resolve_target_to_make(self.topobjdir,
path_arg.relpath())
if make_dir is None and make_target is None:
return 1
# See bug 886162 - we don't want to "accidentally" build
# the entire tree (if that's really the intent, it's
# unlikely they would have specified a directory.)
if not make_dir and not make_target:
print("The specified directory doesn't contain a "
"Makefile and the first parent with one is the "
"root of the tree. Please specify a directory "
"with a Makefile or run |mach build| if you "
"want to build the entire tree.")
return 1
target_pairs.append((make_dir, make_target))
# Possibly add extra make depencies using dumbmake.
if not disable_extra_make_dependencies:
from dumbmake.dumbmake import (dependency_map,
add_extra_dependencies)
depfile = os.path.join(self.topsrcdir, 'build',
'dumbmake-dependencies')
with open(depfile) as f:
dm = dependency_map(f.readlines())
new_pairs = list(add_extra_dependencies(target_pairs, dm))
self.log(logging.DEBUG, 'dumbmake',
{'target_pairs': target_pairs,
'new_pairs': new_pairs},
'Added extra dependencies: will build {new_pairs} ' +
'instead of {target_pairs}.')
target_pairs = new_pairs
# Ensure build backend is up to date. The alternative is to
# have rules in the invoked Makefile to rebuild the build
# backend. But that involves make reinvoking itself and there
# are undesired side-effects of this. See bug 877308 for a
# comprehensive history lesson.
self._run_make(directory=self.topobjdir,
target='backend.RecursiveMakeBackend',
line_handler=output.on_line, log=False,
print_directory=False)
# Build target pairs.
for make_dir, make_target in target_pairs:
# We don't display build status messages during partial
# tree builds because they aren't reliable there. This
# could potentially be fixed if the build monitor were more
# intelligent about encountering undefined state.
status = self._run_make(directory=make_dir, target=make_target,
line_handler=output.on_line, log=False, print_directory=False,
ensure_exit_code=False, num_jobs=jobs, silent=not verbose,
append_env={b'NO_BUILDSTATUS_MESSAGES': b'1'})
if status != 0:
break
else:
monitor.start_resource_recording()
status = self._run_make(srcdir=True, filename='client.mk',
line_handler=output.on_line, log=False, print_directory=False,
allow_parallel=False, ensure_exit_code=False, num_jobs=jobs,
silent=not verbose)
make_extra = self.mozconfig['make_extra'] or []
make_extra = dict(m.split('=', 1) for m in make_extra)
# For universal builds, we need to run the automation steps in
# the first architecture from MOZ_BUILD_PROJECTS
projects = make_extra.get('MOZ_BUILD_PROJECTS')
if projects:
subdir = os.path.join(self.topobjdir, projects.split()[0])
else:
subdir = self.topobjdir
moz_automation = os.getenv('MOZ_AUTOMATION') or make_extra.get('export MOZ_AUTOMATION', None)
if moz_automation and status == 0:
status = self._run_make(target='automation/build', directory=subdir,
line_handler=output.on_line, log=False, print_directory=False,
ensure_exit_code=False, num_jobs=jobs, silent=not verbose)
self.log(logging.WARNING, 'warning_summary',
{'count': len(monitor.warnings_database)},
'{count} compiler warnings present.')
monitor.finish(record_usage=status==0)
high_finder, finder_percent = monitor.have_high_finder_usage()
if high_finder:
print(FINDER_SLOW_MESSAGE % finder_percent)
ccache_end = monitor.ccache_stats()
if ccache_start and ccache_end:
ccache_diff = ccache_end - ccache_start
if ccache_diff:
self.log(logging.INFO, 'ccache',
{'msg': ccache_diff.hit_rate_message()}, "{msg}")
if monitor.elapsed > 300:
# Display a notification when the build completes.
self.notify('Build complete')
if status:
return status
long_build = monitor.elapsed > 600
if long_build:
output.on_line('We know it took a while, but your build finally finished successfully!')
else:
output.on_line('Your build was successful!')
if monitor.have_resource_usage:
excessive, swap_in, swap_out = monitor.have_excessive_swapping()
# if excessive:
# print(EXCESSIVE_SWAP_MESSAGE)
print('To view resource usage of the build, run |mach '
'resource-usage|.')
# Only for full builds because incremental builders likely don't
# need to be burdened with this.
if not what:
try:
# Fennec doesn't have useful output from just building. We should
# arguably make the build action useful for Fennec. Another day...
if self.substs['MOZ_BUILD_APP'] != 'mobile/android':
print('To take your build for a test drive, run: |mach run|')
app = self.substs['MOZ_BUILD_APP']
if app in ('browser', 'mobile/android'):
print('For more information on what to do now, see '
'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox')
except Exception:
# Ignore Exceptions in case we can't find config.status (such
# as when doing OSX Universal builds)
pass
return status
def build(self, what=None, disable_extra_make_dependencies=None, jobs=0):
# This code is only meant to be temporary until the more robust tree
# building code in bug 780329 lands.
from mozbuild.compilation.warnings import WarningsCollector
from mozbuild.compilation.warnings import WarningsDatabase
from mozbuild.util import resolve_target_to_make
warnings_path = self._get_state_filename('warnings.json')
warnings_database = WarningsDatabase()
if os.path.exists(warnings_path):
try:
warnings_database.load_from_file(warnings_path)
except ValueError:
os.remove(warnings_path)
warnings_collector = WarningsCollector(database=warnings_database,
objdir=self.topobjdir)
def on_line(line):
try:
warning = warnings_collector.process_line(line)
if warning:
self.log(logging.INFO, 'compiler_warning', warning,
'Warning: {flag} in {filename}: {message}')
except:
# This will get logged in the more robust implementation.
pass
self.log(logging.INFO, 'build_output', {'line': line}, '{line}')
finder_start_cpu = self._get_finder_cpu_usage()
time_start = time.time()
if what:
top_make = os.path.join(self.topobjdir, 'Makefile')
if not os.path.exists(top_make):
print('Your tree has not been configured yet. Please run '
'|mach build| with no arguments.')
return 1
# Collect target pairs.
target_pairs = []
for target in what:
path_arg = self._wrap_path_argument(target)
make_dir, make_target = resolve_target_to_make(self.topobjdir,
path_arg.relpath())
if make_dir is None and make_target is None:
return 1
target_pairs.append((make_dir, make_target))
# Possibly add extra make depencies using dumbmake.
if not disable_extra_make_dependencies:
from dumbmake.dumbmake import (dependency_map,
add_extra_dependencies)
depfile = os.path.join(self.topsrcdir, 'build',
'dumbmake-dependencies')
with open(depfile) as f:
dm = dependency_map(f.readlines())
new_pairs = list(add_extra_dependencies(target_pairs, dm))
self.log(logging.DEBUG, 'dumbmake',
{'target_pairs': target_pairs,
'new_pairs': new_pairs},
'Added extra dependencies: will build {new_pairs} ' +
'instead of {target_pairs}.')
target_pairs = new_pairs
# Build target pairs.
for make_dir, make_target in target_pairs:
status = self._run_make(directory=make_dir, target=make_target,
line_handler=on_line, log=False, print_directory=False,
ensure_exit_code=False, num_jobs=jobs)
if status != 0:
break
else:
status = self._run_make(srcdir=True, filename='client.mk',
line_handler=on_line, log=False, print_directory=False,
allow_parallel=False, ensure_exit_code=False, num_jobs=jobs)
self.log(logging.WARNING, 'warning_summary',
{'count': len(warnings_collector.database)},
'{count} compiler warnings present.')
warnings_database.prune()
warnings_database.save_to_file(warnings_path)
time_end = time.time()
time_elapsed = time_end - time_start
self._handle_finder_cpu_usage(time_elapsed, finder_start_cpu)
long_build = time_elapsed > 600
if status:
return status
if long_build:
print('We know it took a while, but your build finally finished successfully!')
else:
print('Your build was successful!')
# Only for full builds because incremental builders likely don't
# need to be burdened with this.
if not what:
# Fennec doesn't have useful output from just building. We should
# arguably make the build action useful for Fennec. Another day...
if self.substs['MOZ_BUILD_APP'] != 'mobile/android':
app_path = self.get_binary_path('app')
print('To take your build for a test drive, run: %s' % app_path)
app = self.substs['MOZ_BUILD_APP']
if app in ('browser', 'mobile/android'):
print('For more information on what to do now, see '
'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox')
return status
def test_dependency_map(self):
self.assertEqual(dependency_map([]), {})
self.assertEqual(dependency_map(["a"]), {"a": []})
self.assertEqual(dependency_map(["a", "b"]), {"a": [], "b": []})
self.assertEqual(dependency_map(["a", "b", "c"]), {"a": [], "b": [], "c": []})
# indentation
self.assertEqual(dependency_map(["a", "\tb", "a", "\tc"]), {"a": [], "b": ["a"], "c": ["a"]})
self.assertEqual(dependency_map(["a", "\tb", "\t\tc"]), {"a": [], "b": ["a"], "c": ["b", "a"]})
self.assertEqual(dependency_map(["a", "\tb", "\t\tc", "\td", "\te", "f"]), {"a": [], "b": ["a"], "c": ["b", "a"], "d": ["a"], "e": ["a"], "f": []})
# irregular indentation
self.assertEqual(dependency_map(["\ta", "b"]), {"a": [], "b": []})
self.assertEqual(dependency_map(["a", "\t\t\tb", "\t\tc"]), {"a": [], "b": ["a"], "c": ["a"]})
self.assertEqual(dependency_map(["a", "\t\tb", "\t\t\tc", "\t\td", "\te", "f"]), {"a": [], "b": ["a"], "c": ["b", "a"], "d": ["a"], "e": ["a"], "f": []})
# repetitions
self.assertEqual(dependency_map(["a", "\tb", "a", "\tb"]), {"a": [], "b": ["a"]})
self.assertEqual(dependency_map(["a", "\tb", "\t\tc", "b", "\td", "\t\te"]), {"a": [], "b": ["a"], "d": ["b"], "e": ["d", "b"], "c": ["b", "a"]})
# cycles are okay
self.assertEqual(dependency_map(["a", "\tb", "\t\ta"]), {"a": ["b", "a"], "b": ["a"]})
def build(self, what=None, pymake=False,
disable_extra_make_dependencies=None, jobs=0, verbose=False):
import which
from mozbuild.controller.building import BuildMonitor
from mozbuild.util import resolve_target_to_make
self.log_manager.register_structured_logger(logging.getLogger('mozbuild'))
warnings_path = self._get_state_filename('warnings.json')
monitor = self._spawn(BuildMonitor)
monitor.init(warnings_path)
with BuildOutputManager(self.log_manager, monitor) as output:
monitor.start()
if what:
top_make = os.path.join(self.topobjdir, 'Makefile')
if not os.path.exists(top_make):
print('Your tree has not been configured yet. Please run '
'|mach build| with no arguments.')
return 1
# Collect target pairs.
target_pairs = []
for target in what:
path_arg = self._wrap_path_argument(target)
make_dir, make_target = resolve_target_to_make(self.topobjdir,
path_arg.relpath())
if make_dir is None and make_target is None:
return 1
# See bug 886162 - we don't want to "accidentally" build
# the entire tree (if that's really the intent, it's
# unlikely they would have specified a directory.)
if not make_dir and not make_target:
print("The specified directory doesn't contain a "
"Makefile and the first parent with one is the "
"root of the tree. Please specify a directory "
"with a Makefile or run |mach build| if you "
"want to build the entire tree.")
return 1
target_pairs.append((make_dir, make_target))
# Possibly add extra make depencies using dumbmake.
if not disable_extra_make_dependencies:
from dumbmake.dumbmake import (dependency_map,
add_extra_dependencies)
depfile = os.path.join(self.topsrcdir, 'build',
'dumbmake-dependencies')
with open(depfile) as f:
dm = dependency_map(f.readlines())
new_pairs = list(add_extra_dependencies(target_pairs, dm))
self.log(logging.DEBUG, 'dumbmake',
{'target_pairs': target_pairs,
'new_pairs': new_pairs},
'Added extra dependencies: will build {new_pairs} ' +
'instead of {target_pairs}.')
target_pairs = new_pairs
# Ensure build backend is up to date. The alternative is to
# have rules in the invoked Makefile to rebuild the build
# backend. But that involves make reinvoking itself and there
# are undesired side-effects of this. See bug 877308 for a
# comprehensive history lesson.
self._run_make(directory=self.topobjdir,
target='backend.RecursiveMakeBackend',
force_pymake=pymake, line_handler=output.on_line,
log=False, print_directory=False)
# Build target pairs.
for make_dir, make_target in target_pairs:
# We don't display build status messages during partial
# tree builds because they aren't reliable there. This
# could potentially be fixed if the build monitor were more
# intelligent about encountering undefined state.
status = self._run_make(directory=make_dir, target=make_target,
line_handler=output.on_line, log=False, print_directory=False,
ensure_exit_code=False, num_jobs=jobs, silent=not verbose,
append_env={b'NO_BUILDSTATUS_MESSAGES': b'1'},
force_pymake=pymake)
if status != 0:
break
else:
monitor.start_resource_recording()
status = self._run_make(srcdir=True, filename='client.mk',
line_handler=output.on_line, log=False, print_directory=False,
allow_parallel=False, ensure_exit_code=False, num_jobs=jobs,
silent=not verbose, force_pymake=pymake)
self.log(logging.WARNING, 'warning_summary',
{'count': len(monitor.warnings_database)},
'{count} compiler warnings present.')
monitor.finish(record_usage=status==0)
high_finder, finder_percent = monitor.have_high_finder_usage()
if high_finder:
print(FINDER_SLOW_MESSAGE % finder_percent)
if monitor.elapsed > 300:
# Display a notification when the build completes.
# This could probably be uplifted into the mach core or at least
# into a helper API. It is here as an experimentation to see how it
# is received.
try:
if sys.platform.startswith('darwin'):
notifier = which.which('terminal-notifier')
self.run_process([notifier, '-title',
'Mozilla Build System', '-group', 'mozbuild',
'-message', 'Build complete'], ensure_exit_code=False)
except which.WhichError:
pass
except Exception as e:
self.log(logging.WARNING, 'notifier-failed', {'error':
e.message}, 'Notification center failed: {error}')
if status:
return status
long_build = monitor.elapsed > 600
if long_build:
print('We know it took a while, but your build finally finished successfully!')
else:
print('Your build was successful!')
if monitor.have_resource_usage:
excessive, swap_in, swap_out = monitor.have_excessive_swapping()
# if excessive:
# print(EXCESSIVE_SWAP_MESSAGE)
print('To view resource usage of the build, run |mach '
'resource-usage|.')
# Only for full builds because incremental builders likely don't
# need to be burdened with this.
if not what:
# Fennec doesn't have useful output from just building. We should
# arguably make the build action useful for Fennec. Another day...
if self.substs['MOZ_BUILD_APP'] != 'mobile/android':
app_path = self.get_binary_path('app')
print('To take your build for a test drive, run: %s' % app_path)
app = self.substs['MOZ_BUILD_APP']
if app in ('browser', 'mobile/android'):
print('For more information on what to do now, see '
'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox')
return status
def build(self,
what=None,
pymake=False,
disable_extra_make_dependencies=None,
jobs=0,
verbose=False):
import which
from mozbuild.controller.building import BuildMonitor
from mozbuild.util import resolve_target_to_make
self.log_manager.register_structured_logger(
logging.getLogger('mozbuild'))
warnings_path = self._get_state_filename('warnings.json')
monitor = self._spawn(BuildMonitor)
monitor.init(warnings_path)
with BuildOutputManager(self.log_manager, monitor) as output:
monitor.start()
if what:
top_make = os.path.join(self.topobjdir, 'Makefile')
if not os.path.exists(top_make):
print('Your tree has not been configured yet. Please run '
'|mach build| with no arguments.')
return 1
# Collect target pairs.
target_pairs = []
for target in what:
path_arg = self._wrap_path_argument(target)
make_dir, make_target = resolve_target_to_make(
self.topobjdir, path_arg.relpath())
if make_dir is None and make_target is None:
return 1
# See bug 886162 - we don't want to "accidentally" build
# the entire tree (if that's really the intent, it's
# unlikely they would have specified a directory.)
if not make_dir and not make_target:
print("The specified directory doesn't contain a "
"Makefile and the first parent with one is the "
"root of the tree. Please specify a directory "
"with a Makefile or run |mach build| if you "
"want to build the entire tree.")
return 1
target_pairs.append((make_dir, make_target))
# Possibly add extra make depencies using dumbmake.
if not disable_extra_make_dependencies:
from dumbmake.dumbmake import (dependency_map,
add_extra_dependencies)
depfile = os.path.join(self.topsrcdir, 'build',
'dumbmake-dependencies')
with open(depfile) as f:
dm = dependency_map(f.readlines())
new_pairs = list(add_extra_dependencies(target_pairs, dm))
self.log(
logging.DEBUG, 'dumbmake', {
'target_pairs': target_pairs,
'new_pairs': new_pairs
},
'Added extra dependencies: will build {new_pairs} ' +
'instead of {target_pairs}.')
target_pairs = new_pairs
# Ensure build backend is up to date. The alternative is to
# have rules in the invoked Makefile to rebuild the build
# backend. But that involves make reinvoking itself and there
# are undesired side-effects of this. See bug 877308 for a
# comprehensive history lesson.
self._run_make(directory=self.topobjdir,
target='backend.RecursiveMakeBackend',
force_pymake=pymake,
line_handler=output.on_line,
log=False,
print_directory=False)
# Build target pairs.
for make_dir, make_target in target_pairs:
# We don't display build status messages during partial
# tree builds because they aren't reliable there. This
# could potentially be fixed if the build monitor were more
# intelligent about encountering undefined state.
status = self._run_make(
directory=make_dir,
target=make_target,
line_handler=output.on_line,
log=False,
print_directory=False,
ensure_exit_code=False,
num_jobs=jobs,
silent=not verbose,
append_env={b'NO_BUILDSTATUS_MESSAGES': b'1'},
force_pymake=pymake)
if status != 0:
break
else:
monitor.start_resource_recording()
status = self._run_make(srcdir=True,
filename='client.mk',
line_handler=output.on_line,
log=False,
print_directory=False,
allow_parallel=False,
ensure_exit_code=False,
num_jobs=jobs,
silent=not verbose,
force_pymake=pymake)
self.log(logging.WARNING, 'warning_summary',
{'count': len(monitor.warnings_database)},
'{count} compiler warnings present.')
monitor.finish(record_usage=status == 0)
high_finder, finder_percent = monitor.have_high_finder_usage()
if high_finder:
print(FINDER_SLOW_MESSAGE % finder_percent)
if monitor.elapsed > 300:
# Display a notification when the build completes.
# This could probably be uplifted into the mach core or at least
# into a helper API. It is here as an experimentation to see how it
# is received.
try:
if sys.platform.startswith('darwin'):
notifier = which.which('terminal-notifier')
self.run_process([
notifier, '-title', 'Mozilla Build System', '-group',
'mozbuild', '-message', 'Build complete'
],
ensure_exit_code=False)
except which.WhichError:
pass
except Exception as e:
self.log(logging.WARNING, 'notifier-failed',
{'error': e.message},
'Notification center failed: {error}')
if status:
return status
long_build = monitor.elapsed > 600
if long_build:
print(
'We know it took a while, but your build finally finished successfully!'
)
else:
print('Your build was successful!')
if monitor.have_resource_usage:
excessive, swap_in, swap_out = monitor.have_excessive_swapping()
# if excessive:
# print(EXCESSIVE_SWAP_MESSAGE)
print('To view resource usage of the build, run |mach '
'resource-usage|.')
# Only for full builds because incremental builders likely don't
# need to be burdened with this.
if not what:
# Fennec doesn't have useful output from just building. We should
# arguably make the build action useful for Fennec. Another day...
if self.substs['MOZ_BUILD_APP'] != 'mobile/android':
print('To take your build for a test drive, run: |mach run|')
app = self.substs['MOZ_BUILD_APP']
if app in ('browser', 'mobile/android'):
print(
'For more information on what to do now, see '
'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox'
)
return status
def build(self,
what=None,
disable_extra_make_dependencies=None,
jobs=0,
directory=None,
verbose=False,
keep_going=False,
mach_context=None):
"""Invoke the build backend.
``what`` defines the thing to build. If not defined, the default
target is used.
"""
warnings_path = self._get_state_filename('warnings.json')
monitor = self._spawn(BuildMonitor)
monitor.init(warnings_path)
ccache_start = monitor.ccache_stats()
footer = BuildProgressFooter(self.log_manager.terminal, monitor)
# Disable indexing in objdir because it is not necessary and can slow
# down builds.
mkdir(self.topobjdir, not_indexed=True)
with BuildOutputManager(self.log_manager, monitor, footer) as output:
monitor.start()
if directory is not None and not what:
print('Can only use -C/--directory with an explicit target '
'name.')
return 1
if directory is not None:
disable_extra_make_dependencies = True
directory = mozpath.normsep(directory)
if directory.startswith('/'):
directory = directory[1:]
status = None
monitor.start_resource_recording()
if what:
top_make = os.path.join(self.topobjdir, 'Makefile')
if not os.path.exists(top_make):
print('Your tree has not been configured yet. Please run '
'|mach build| with no arguments.')
return 1
# Collect target pairs.
target_pairs = []
for target in what:
path_arg = self._wrap_path_argument(target)
if directory is not None:
make_dir = os.path.join(self.topobjdir, directory)
make_target = target
else:
make_dir, make_target = \
resolve_target_to_make(self.topobjdir,
path_arg.relpath())
if make_dir is None and make_target is None:
return 1
# See bug 886162 - we don't want to "accidentally" build
# the entire tree (if that's really the intent, it's
# unlikely they would have specified a directory.)
if not make_dir and not make_target:
print("The specified directory doesn't contain a "
"Makefile and the first parent with one is the "
"root of the tree. Please specify a directory "
"with a Makefile or run |mach build| if you "
"want to build the entire tree.")
return 1
target_pairs.append((make_dir, make_target))
# Possibly add extra make depencies using dumbmake.
if not disable_extra_make_dependencies:
from dumbmake.dumbmake import (dependency_map,
add_extra_dependencies)
depfile = os.path.join(self.topsrcdir, 'build',
'dumbmake-dependencies')
with open(depfile) as f:
dm = dependency_map(f.readlines())
new_pairs = list(add_extra_dependencies(target_pairs, dm))
self.log(
logging.DEBUG, 'dumbmake', {
'target_pairs': target_pairs,
'new_pairs': new_pairs
},
'Added extra dependencies: will build {new_pairs} ' +
'instead of {target_pairs}.')
target_pairs = new_pairs
# Ensure build backend is up to date. The alternative is to
# have rules in the invoked Makefile to rebuild the build
# backend. But that involves make reinvoking itself and there
# are undesired side-effects of this. See bug 877308 for a
# comprehensive history lesson.
self._run_make(directory=self.topobjdir,
target='backend',
line_handler=output.on_line,
log=False,
print_directory=False,
keep_going=keep_going)
# Build target pairs.
for make_dir, make_target in target_pairs:
# We don't display build status messages during partial
# tree builds because they aren't reliable there. This
# could potentially be fixed if the build monitor were more
# intelligent about encountering undefined state.
status = self._run_make(
directory=make_dir,
target=make_target,
line_handler=output.on_line,
log=False,
print_directory=False,
ensure_exit_code=False,
num_jobs=jobs,
silent=not verbose,
append_env={b'NO_BUILDSTATUS_MESSAGES': b'1'},
keep_going=keep_going)
if status != 0:
break
else:
# Try to call the default backend's build() method. This will
# run configure to determine BUILD_BACKENDS if it hasn't run
# yet.
config = None
try:
config = self.config_environment
except Exception:
config_rc = self.configure(buildstatus_messages=True,
line_handler=output.on_line)
if config_rc != 0:
return config_rc
# Even if configure runs successfully, we may have trouble
# getting the config_environment for some builds, such as
# OSX Universal builds. These have to go through client.mk
# regardless.
try:
config = self.config_environment
except Exception:
pass
if config:
active_backend = config.substs.get('BUILD_BACKENDS',
[None])[0]
if active_backend:
backend_cls = get_backend_class(active_backend)(config)
status = backend_cls.build(self, output, jobs, verbose)
# If the backend doesn't specify a build() method, then just
# call client.mk directly.
if status is None:
status = self._run_make(srcdir=True,
filename='client.mk',
line_handler=output.on_line,
log=False,
print_directory=False,
allow_parallel=False,
ensure_exit_code=False,
num_jobs=jobs,
silent=not verbose,
keep_going=keep_going)
self.log(logging.WARNING, 'warning_summary',
{'count': len(monitor.warnings_database)},
'{count} compiler warnings present.')
monitor.finish(record_usage=status == 0)
# Print the collected compiler warnings. This is redundant with
# inline output from the compiler itself. However, unlike inline
# output, this list is sorted and grouped by file, making it
# easier to triage output.
#
# Only do this if we had a successful build. If the build failed,
# there are more important things in the log to look for than
# whatever code we warned about.
if not status:
# Suppress warnings for 3rd party projects in local builds
# until we suppress them for real.
# TODO remove entries/feature once we stop generating warnings
# in these directories.
pathToThirdparty = os.path.join(self.topsrcdir, "tools",
"rewriting", "ThirdPartyPaths.txt")
if os.path.exists(pathToThirdparty):
with open(pathToThirdparty) as f:
# Normalize the path (no trailing /)
LOCAL_SUPPRESS_DIRS = tuple(
d.rstrip('/') for d in f.read().splitlines())
else:
# For application based on gecko like thunderbird
LOCAL_SUPPRESS_DIRS = ()
suppressed_by_dir = Counter()
for warning in sorted(monitor.instance_warnings):
path = mozpath.normsep(warning['filename'])
if path.startswith(self.topsrcdir):
path = path[len(self.topsrcdir) + 1:]
warning['normpath'] = path
if (path.startswith(LOCAL_SUPPRESS_DIRS)
and 'MOZ_AUTOMATION' not in os.environ):
for d in LOCAL_SUPPRESS_DIRS:
if path.startswith(d):
suppressed_by_dir[d] += 1
break
continue
if warning['column'] is not None:
self.log(
logging.WARNING, 'compiler_warning', warning,
'warning: {normpath}:{line}:{column} [{flag}] '
'{message}')
else:
self.log(logging.WARNING, 'compiler_warning', warning,
'warning: {normpath}:{line} [{flag}] {message}')
for d, count in sorted(suppressed_by_dir.items()):
self.log(logging.WARNING, 'suppressed_warning', {
'dir': d,
'count': count
}, '(suppressed {count} warnings in {dir})')
high_finder, finder_percent = monitor.have_high_finder_usage()
if high_finder:
print(FINDER_SLOW_MESSAGE % finder_percent)
ccache_end = monitor.ccache_stats()
ccache_diff = None
if ccache_start and ccache_end:
ccache_diff = ccache_end - ccache_start
if ccache_diff:
self.log(logging.INFO, 'ccache',
{'msg': ccache_diff.hit_rate_message()}, "{msg}")
notify_minimum_time = 300
try:
notify_minimum_time = int(
os.environ.get('MACH_NOTIFY_MINTIME', '300'))
except ValueError:
# Just stick with the default
pass
if monitor.elapsed > notify_minimum_time:
# Display a notification when the build completes.
self.notify('Build complete' if not status else 'Build failed')
if status:
return status
long_build = monitor.elapsed > 600
if long_build:
output.on_line(
'We know it took a while, but your build finally finished successfully!'
)
else:
output.on_line('Your build was successful!')
if monitor.have_resource_usage:
excessive, swap_in, swap_out = monitor.have_excessive_swapping()
# if excessive:
# print(EXCESSIVE_SWAP_MESSAGE)
print('To view resource usage of the build, run |mach '
'resource-usage|.')
telemetry_handler = getattr(mach_context, 'telemetry_handler',
None)
telemetry_data = monitor.get_resource_usage()
# Record build configuration data. For now, we cherry pick
# items we need rather than grabbing everything, in order
# to avoid accidentally disclosing PII.
telemetry_data['substs'] = {}
try:
for key in [
'MOZ_ARTIFACT_BUILDS', 'MOZ_USING_CCACHE',
'MOZ_USING_SCCACHE'
]:
value = self.substs.get(key, False)
telemetry_data['substs'][key] = value
except BuildEnvironmentNotFoundException:
pass
# Grab ccache stats if available. We need to be careful not
# to capture information that can potentially identify the
# user (such as the cache location)
if ccache_diff:
telemetry_data['ccache'] = {}
for key in [key[0] for key in ccache_diff.STATS_KEYS]:
try:
telemetry_data['ccache'][key] = ccache_diff._values[
key]
except KeyError:
pass
if telemetry_handler:
telemetry_handler(mach_context, telemetry_data)
# Only for full builds because incremental builders likely don't
# need to be burdened with this.
if not what:
try:
# Fennec doesn't have useful output from just building. We should
# arguably make the build action useful for Fennec. Another day...
if self.substs['MOZ_BUILD_APP'] != 'mobile/android':
print(
'To take your build for a test drive, run: |mach run|')
app = self.substs['MOZ_BUILD_APP']
if app in ('browser', 'mobile/android'):
print(
'For more information on what to do now, see '
'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox'
)
except Exception:
# Ignore Exceptions in case we can't find config.status (such
# as when doing OSX Universal builds)
pass
return status
def build(self, what=None, disable_extra_make_dependencies=None, jobs=0,
directory=None, verbose=False):
import which
from mozbuild.controller.building import BuildMonitor
from mozbuild.util import resolve_target_to_make
self.log_manager.register_structured_logger(logging.getLogger('mozbuild'))
warnings_path = self._get_state_filename('warnings.json')
monitor = self._spawn(BuildMonitor)
monitor.init(warnings_path)
ccache_start = monitor.ccache_stats()
with BuildOutputManager(self.log_manager, monitor) as output:
monitor.start()
if directory is not None and not what:
print('Can only use -C/--directory with an explicit target '
'name.')
return 1
if directory is not None:
disable_extra_make_dependencies=True
directory = mozpath.normsep(directory)
if directory.startswith('/'):
directory = directory[1:]
if what:
top_make = os.path.join(self.topobjdir, 'Makefile')
if not os.path.exists(top_make):
print('Your tree has not been configured yet. Please run '
'|mach build| with no arguments.')
return 1
# Collect target pairs.
target_pairs = []
for target in what:
path_arg = self._wrap_path_argument(target)
if directory is not None:
make_dir = os.path.join(self.topobjdir, directory)
make_target = target
else:
make_dir, make_target = \
resolve_target_to_make(self.topobjdir,
path_arg.relpath())
if make_dir is None and make_target is None:
return 1
# See bug 886162 - we don't want to "accidentally" build
# the entire tree (if that's really the intent, it's
# unlikely they would have specified a directory.)
if not make_dir and not make_target:
print("The specified directory doesn't contain a "
"Makefile and the first parent with one is the "
"root of the tree. Please specify a directory "
"with a Makefile or run |mach build| if you "
"want to build the entire tree.")
return 1
target_pairs.append((make_dir, make_target))
# Possibly add extra make depencies using dumbmake.
if not disable_extra_make_dependencies:
from dumbmake.dumbmake import (dependency_map,
add_extra_dependencies)
depfile = os.path.join(self.topsrcdir, 'build',
'dumbmake-dependencies')
with open(depfile) as f:
dm = dependency_map(f.readlines())
new_pairs = list(add_extra_dependencies(target_pairs, dm))
self.log(logging.DEBUG, 'dumbmake',
{'target_pairs': target_pairs,
'new_pairs': new_pairs},
'Added extra dependencies: will build {new_pairs} ' +
'instead of {target_pairs}.')
target_pairs = new_pairs
# Ensure build backend is up to date. The alternative is to
# have rules in the invoked Makefile to rebuild the build
# backend. But that involves make reinvoking itself and there
# are undesired side-effects of this. See bug 877308 for a
# comprehensive history lesson.
self._run_make(directory=self.topobjdir,
target='backend.RecursiveMakeBackend',
line_handler=output.on_line, log=False,
print_directory=False)
# Build target pairs.
for make_dir, make_target in target_pairs:
# We don't display build status messages during partial
# tree builds because they aren't reliable there. This
# could potentially be fixed if the build monitor were more
# intelligent about encountering undefined state.
status = self._run_make(directory=make_dir, target=make_target,
line_handler=output.on_line, log=False, print_directory=False,
ensure_exit_code=False, num_jobs=jobs, silent=not verbose,
append_env={b'NO_BUILDSTATUS_MESSAGES': b'1'})
if status != 0:
break
else:
monitor.start_resource_recording()
status = self._run_make(srcdir=True, filename='client.mk',
line_handler=output.on_line, log=False, print_directory=False,
allow_parallel=False, ensure_exit_code=False, num_jobs=jobs,
silent=not verbose)
make_extra = self.mozconfig['make_extra'] or []
make_extra = dict(m.split('=', 1) for m in make_extra)
# For universal builds, we need to run the automation steps in
# the first architecture from MOZ_BUILD_PROJECTS
projects = make_extra.get('MOZ_BUILD_PROJECTS')
if projects:
subdir = os.path.join(self.topobjdir, projects.split()[0])
else:
subdir = self.topobjdir
moz_automation = os.getenv('MOZ_AUTOMATION') or make_extra.get('export MOZ_AUTOMATION', None)
if moz_automation and status == 0:
status = self._run_make(target='automation/build', directory=subdir,
line_handler=output.on_line, log=False, print_directory=False,
ensure_exit_code=False, num_jobs=jobs, silent=not verbose)
self.log(logging.WARNING, 'warning_summary',
{'count': len(monitor.warnings_database)},
'{count} compiler warnings present.')
monitor.finish(record_usage=status==0)
high_finder, finder_percent = monitor.have_high_finder_usage()
if high_finder:
print(FINDER_SLOW_MESSAGE % finder_percent)
ccache_end = monitor.ccache_stats()
if ccache_start and ccache_end:
ccache_diff = ccache_end - ccache_start
if ccache_diff:
self.log(logging.INFO, 'ccache',
{'msg': ccache_diff.hit_rate_message()}, "{msg}")
moz_nospam = os.environ.get('MOZ_NOSPAM')
if monitor.elapsed > 300 and not moz_nospam:
# Display a notification when the build completes.
# This could probably be uplifted into the mach core or at least
# into a helper API. It is here as an experimentation to see how it
# is received.
try:
if sys.platform.startswith('darwin'):
try:
notifier = which.which('terminal-notifier')
except which.WhichError:
raise Exception('Install terminal-notifier to get '
'a notification when the build finishes.')
self.run_process([notifier, '-title',
'Mozilla Build System', '-group', 'mozbuild',
'-message', 'Build complete'], ensure_exit_code=False)
elif sys.platform.startswith('linux'):
try:
import dbus
except ImportError:
raise Exception('Install the python dbus module to '
'get a notification when the build finishes.')
bus = dbus.SessionBus()
notify = bus.get_object('org.freedesktop.Notifications',
'/org/freedesktop/Notifications')
method = notify.get_dbus_method('Notify',
'org.freedesktop.Notifications')
method('Mozilla Build System', 0, '', 'Build complete', '', [], [], -1)
elif sys.platform.startswith('win'):
from ctypes import Structure, windll, POINTER, sizeof
from ctypes.wintypes import DWORD, HANDLE, WINFUNCTYPE, BOOL, UINT
class FLASHWINDOW(Structure):
_fields_ = [("cbSize", UINT),
("hwnd", HANDLE),
("dwFlags", DWORD),
("uCount", UINT),
("dwTimeout", DWORD)]
FlashWindowExProto = WINFUNCTYPE(BOOL, POINTER(FLASHWINDOW))
FlashWindowEx = FlashWindowExProto(("FlashWindowEx", windll.user32))
FLASHW_CAPTION = 0x01
FLASHW_TRAY = 0x02
FLASHW_TIMERNOFG = 0x0C
params = FLASHWINDOW(sizeof(FLASHWINDOW),
windll.kernel32.GetConsoleWindow(),
FLASHW_CAPTION | FLASHW_TRAY | FLASHW_TIMERNOFG, 3, 0)
FlashWindowEx(params)
except Exception as e:
self.log(logging.WARNING, 'notifier-failed', {'error':
e.message}, 'Notification center failed: {error}')
if status:
return status
long_build = monitor.elapsed > 600
if long_build:
output.on_line('We know it took a while, but your build finally finished successfully!')
else:
output.on_line('Your build was successful!')
if monitor.have_resource_usage:
excessive, swap_in, swap_out = monitor.have_excessive_swapping()
# if excessive:
# print(EXCESSIVE_SWAP_MESSAGE)
print('To view resource usage of the build, run |mach '
'resource-usage|.')
# Only for full builds because incremental builders likely don't
# need to be burdened with this.
if not what:
try:
# Fennec doesn't have useful output from just building. We should
# arguably make the build action useful for Fennec. Another day...
if self.substs['MOZ_BUILD_APP'] != 'mobile/android':
print('To take your build for a test drive, run: |mach run|')
app = self.substs['MOZ_BUILD_APP']
if app in ('browser', 'mobile/android'):
print('For more information on what to do now, see '
'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox')
except Exception:
# Ignore Exceptions in case we can't find config.status (such
# as when doing OSX Universal builds)
pass
return status
def build(self,
what=None,
disable_extra_make_dependencies=None,
jobs=0,
verbose=False):
import which
from mozbuild.controller.building import BuildMonitor
from mozbuild.util import resolve_target_to_make
self.log_manager.register_structured_logger(
logging.getLogger('mozbuild'))
warnings_path = self._get_state_filename('warnings.json')
monitor = self._spawn(BuildMonitor)
monitor.init(warnings_path)
ccache_start = monitor.ccache_stats()
with BuildOutputManager(self.log_manager, monitor) as output:
monitor.start()
if what:
top_make = os.path.join(self.topobjdir, 'Makefile')
if not os.path.exists(top_make):
print('Your tree has not been configured yet. Please run '
'|mach build| with no arguments.')
return 1
# Collect target pairs.
target_pairs = []
for target in what:
path_arg = self._wrap_path_argument(target)
make_dir, make_target = resolve_target_to_make(
self.topobjdir, path_arg.relpath())
if make_dir is None and make_target is None:
return 1
# See bug 886162 - we don't want to "accidentally" build
# the entire tree (if that's really the intent, it's
# unlikely they would have specified a directory.)
if not make_dir and not make_target:
print("The specified directory doesn't contain a "
"Makefile and the first parent with one is the "
"root of the tree. Please specify a directory "
"with a Makefile or run |mach build| if you "
"want to build the entire tree.")
return 1
target_pairs.append((make_dir, make_target))
# Possibly add extra make depencies using dumbmake.
if not disable_extra_make_dependencies:
from dumbmake.dumbmake import (dependency_map,
add_extra_dependencies)
depfile = os.path.join(self.topsrcdir, 'build',
'dumbmake-dependencies')
with open(depfile) as f:
dm = dependency_map(f.readlines())
new_pairs = list(add_extra_dependencies(target_pairs, dm))
self.log(
logging.DEBUG, 'dumbmake', {
'target_pairs': target_pairs,
'new_pairs': new_pairs
},
'Added extra dependencies: will build {new_pairs} ' +
'instead of {target_pairs}.')
target_pairs = new_pairs
# Ensure build backend is up to date. The alternative is to
# have rules in the invoked Makefile to rebuild the build
# backend. But that involves make reinvoking itself and there
# are undesired side-effects of this. See bug 877308 for a
# comprehensive history lesson.
self._run_make(directory=self.topobjdir,
target='backend.RecursiveMakeBackend',
line_handler=output.on_line,
log=False,
print_directory=False)
# Build target pairs.
for make_dir, make_target in target_pairs:
# We don't display build status messages during partial
# tree builds because they aren't reliable there. This
# could potentially be fixed if the build monitor were more
# intelligent about encountering undefined state.
status = self._run_make(
directory=make_dir,
target=make_target,
line_handler=output.on_line,
log=False,
print_directory=False,
ensure_exit_code=False,
num_jobs=jobs,
silent=not verbose,
append_env={b'NO_BUILDSTATUS_MESSAGES': b'1'})
if status != 0:
break
else:
monitor.start_resource_recording()
status = self._run_make(srcdir=True,
filename='client.mk',
line_handler=output.on_line,
log=False,
print_directory=False,
allow_parallel=False,
ensure_exit_code=False,
num_jobs=jobs,
silent=not verbose)
make_extra = self.mozconfig['make_extra'] or []
make_extra = dict(m.split('=', 1) for m in make_extra)
# For universal builds, we need to run the automation steps in
# the first architecture from MOZ_BUILD_PROJECTS
projects = make_extra.get('MOZ_BUILD_PROJECTS')
if projects:
subdir = os.path.join(self.topobjdir, projects.split()[0])
else:
subdir = self.topobjdir
moz_automation = os.getenv('MOZ_AUTOMATION') or make_extra.get(
'export MOZ_AUTOMATION', None)
if moz_automation and status == 0:
status = self._run_make(target='automation/build',
directory=subdir,
line_handler=output.on_line,
log=False,
print_directory=False,
ensure_exit_code=False,
num_jobs=jobs,
silent=not verbose)
self.log(logging.WARNING, 'warning_summary',
{'count': len(monitor.warnings_database)},
'{count} compiler warnings present.')
monitor.finish(record_usage=status == 0)
high_finder, finder_percent = monitor.have_high_finder_usage()
if high_finder:
print(FINDER_SLOW_MESSAGE % finder_percent)
ccache_end = monitor.ccache_stats()
if ccache_start and ccache_end:
ccache_diff = ccache_end - ccache_start
if ccache_diff:
self.log(logging.INFO, 'ccache',
{'msg': ccache_diff.hit_rate_message()}, "{msg}")
moz_nospam = os.environ.get('MOZ_NOSPAM')
if monitor.elapsed > 300 and not moz_nospam:
# Display a notification when the build completes.
# This could probably be uplifted into the mach core or at least
# into a helper API. It is here as an experimentation to see how it
# is received.
try:
if sys.platform.startswith('darwin'):
try:
notifier = which.which('terminal-notifier')
except which.WhichError:
raise Exception(
'Install terminal-notifier to get '
'a notification when the build finishes.')
self.run_process([
notifier, '-title', 'Mozilla Build System', '-group',
'mozbuild', '-message', 'Build complete'
],
ensure_exit_code=False)
elif sys.platform.startswith('linux'):
try:
import dbus
except ImportError:
raise Exception(
'Install the python dbus module to '
'get a notification when the build finishes.')
bus = dbus.SessionBus()
notify = bus.get_object('org.freedesktop.Notifications',
'/org/freedesktop/Notifications')
method = notify.get_dbus_method(
'Notify', 'org.freedesktop.Notifications')
method('Mozilla Build System', 0, '', 'Build complete', '',
[], [], -1)
elif sys.platform.startswith('win'):
from ctypes import Structure, windll, POINTER, sizeof
from ctypes.wintypes import DWORD, HANDLE, WINFUNCTYPE, BOOL, UINT
class FLASHWINDOW(Structure):
_fields_ = [("cbSize", UINT), ("hwnd", HANDLE),
("dwFlags", DWORD), ("uCount", UINT),
("dwTimeout", DWORD)]
FlashWindowExProto = WINFUNCTYPE(BOOL,
POINTER(FLASHWINDOW))
FlashWindowEx = FlashWindowExProto(
("FlashWindowEx", windll.user32))
FLASHW_CAPTION = 0x01
FLASHW_TRAY = 0x02
FLASHW_TIMERNOFG = 0x0C
params = FLASHWINDOW(
sizeof(FLASHWINDOW),
windll.kernel32.GetConsoleWindow(),
FLASHW_CAPTION | FLASHW_TRAY | FLASHW_TIMERNOFG, 3, 0)
FlashWindowEx(params)
except Exception as e:
self.log(logging.WARNING, 'notifier-failed',
{'error': e.message},
'Notification center failed: {error}')
if status:
return status
long_build = monitor.elapsed > 600
if long_build:
output.on_line(
'We know it took a while, but your build finally finished successfully!'
)
else:
output.on_line('Your build was successful!')
if monitor.have_resource_usage:
excessive, swap_in, swap_out = monitor.have_excessive_swapping()
# if excessive:
# print(EXCESSIVE_SWAP_MESSAGE)
print('To view resource usage of the build, run |mach '
'resource-usage|.')
# Only for full builds because incremental builders likely don't
# need to be burdened with this.
if not what:
try:
# Fennec doesn't have useful output from just building. We should
# arguably make the build action useful for Fennec. Another day...
if self.substs['MOZ_BUILD_APP'] != 'mobile/android':
print(
'To take your build for a test drive, run: |mach run|')
app = self.substs['MOZ_BUILD_APP']
if app in ('browser', 'mobile/android'):
print(
'For more information on what to do now, see '
'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox'
)
except Exception:
# Ignore Exceptions in case we can't find config.status (such
# as when doing OSX Universal builds)
pass
return status
def test_dependency_map(self):
self.assertEqual(dependency_map([]), {})
self.assertEqual(dependency_map(["a"]), {"a": []})
self.assertEqual(dependency_map(["a", "b"]), {"a": [], "b": []})
self.assertEqual(dependency_map(["a", "b", "c"]), {
"a": [],
"b": [],
"c": []
})
# indentation
self.assertEqual(dependency_map(["a", "\tb", "a", "\tc"]), {
"a": [],
"b": ["a"],
"c": ["a"]
})
self.assertEqual(dependency_map(["a", "\tb", "\t\tc"]), {
"a": [],
"b": ["a"],
"c": ["b", "a"]
})
self.assertEqual(
dependency_map(["a", "\tb", "\t\tc", "\td", "\te", "f"]), {
"a": [],
"b": ["a"],
"c": ["b", "a"],
"d": ["a"],
"e": ["a"],
"f": []
})
# irregular indentation
self.assertEqual(dependency_map(["\ta", "b"]), {"a": [], "b": []})
self.assertEqual(dependency_map(["a", "\t\t\tb", "\t\tc"]), {
"a": [],
"b": ["a"],
"c": ["a"]
})
self.assertEqual(
dependency_map(["a", "\t\tb", "\t\t\tc", "\t\td", "\te", "f"]), {
"a": [],
"b": ["a"],
"c": ["b", "a"],
"d": ["a"],
"e": ["a"],
"f": []
})
# repetitions
self.assertEqual(dependency_map(["a", "\tb", "a", "\tb"]), {
"a": [],
"b": ["a"]
})
self.assertEqual(
dependency_map(["a", "\tb", "\t\tc", "b", "\td", "\t\te"]), {
"a": [],
"b": ["a"],
"d": ["b"],
"e": ["d", "b"],
"c": ["b", "a"]
})
# cycles are okay
self.assertEqual(dependency_map(["a", "\tb", "\t\ta"]), {
"a": ["b", "a"],
"b": ["a"]
})
def build(self,
what=None,
disable_extra_make_dependencies=None,
jobs=0,
verbose=False):
from mozbuild.controller.building import BuildMonitor
from mozbuild.util import resolve_target_to_make
warnings_path = self._get_state_filename('warnings.json')
monitor = BuildMonitor(self.topobjdir, warnings_path)
with BuildOutputManager(self.log_manager, monitor) as output:
monitor.start()
if what:
top_make = os.path.join(self.topobjdir, 'Makefile')
if not os.path.exists(top_make):
print('Your tree has not been configured yet. Please run '
'|mach build| with no arguments.')
return 1
# Collect target pairs.
target_pairs = []
for target in what:
path_arg = self._wrap_path_argument(target)
make_dir, make_target = resolve_target_to_make(
self.topobjdir, path_arg.relpath())
if make_dir is None and make_target is None:
return 1
target_pairs.append((make_dir, make_target))
# Possibly add extra make depencies using dumbmake.
if not disable_extra_make_dependencies:
from dumbmake.dumbmake import (dependency_map,
add_extra_dependencies)
depfile = os.path.join(self.topsrcdir, 'build',
'dumbmake-dependencies')
with open(depfile) as f:
dm = dependency_map(f.readlines())
new_pairs = list(add_extra_dependencies(target_pairs, dm))
self.log(
logging.DEBUG, 'dumbmake', {
'target_pairs': target_pairs,
'new_pairs': new_pairs
},
'Added extra dependencies: will build {new_pairs} ' +
'instead of {target_pairs}.')
target_pairs = new_pairs
# Build target pairs.
for make_dir, make_target in target_pairs:
status = self._run_make(directory=make_dir,
target=make_target,
line_handler=output.on_line,
log=False,
print_directory=False,
ensure_exit_code=False,
num_jobs=jobs,
silent=not verbose)
if status != 0:
break
else:
status = self._run_make(srcdir=True,
filename='client.mk',
line_handler=output.on_line,
log=False,
print_directory=False,
allow_parallel=False,
ensure_exit_code=False,
num_jobs=jobs,
silent=not verbose)
self.log(logging.WARNING, 'warning_summary',
{'count': len(monitor.warnings_database)},
'{count} compiler warnings present.')
monitor.finish()
high_finder, finder_percent = monitor.have_high_finder_usage()
if high_finder:
print(FINDER_SLOW_MESSAGE % finder_percent)
long_build = monitor.elapsed > 600
if status:
return status
if long_build:
print(
'We know it took a while, but your build finally finished successfully!'
)
else:
print('Your build was successful!')
# Only for full builds because incremental builders likely don't
# need to be burdened with this.
if not what:
# Fennec doesn't have useful output from just building. We should
# arguably make the build action useful for Fennec. Another day...
if self.substs['MOZ_BUILD_APP'] != 'mobile/android':
app_path = self.get_binary_path('app')
print('To take your build for a test drive, run: %s' %
app_path)
app = self.substs['MOZ_BUILD_APP']
if app in ('browser', 'mobile/android'):
print(
'Please remember that you also need to PACKAGE your build '
'to have all components properly included and unnecessary files removed.'
)
return status
def build(self, what=None, disable_extra_make_dependencies=None, jobs=0,
directory=None, verbose=False, keep_going=False, mach_context=None):
"""Invoke the build backend.
``what`` defines the thing to build. If not defined, the default
target is used.
"""
warnings_path = self._get_state_filename('warnings.json')
monitor = self._spawn(BuildMonitor)
monitor.init(warnings_path)
ccache_start = monitor.ccache_stats()
footer = BuildProgressFooter(self.log_manager.terminal, monitor)
# Disable indexing in objdir because it is not necessary and can slow
# down builds.
mkdir(self.topobjdir, not_indexed=True)
with BuildOutputManager(self.log_manager, monitor, footer) as output:
monitor.start()
if directory is not None and not what:
print('Can only use -C/--directory with an explicit target '
'name.')
return 1
if directory is not None:
disable_extra_make_dependencies=True
directory = mozpath.normsep(directory)
if directory.startswith('/'):
directory = directory[1:]
status = None
monitor.start_resource_recording()
if what:
top_make = os.path.join(self.topobjdir, 'Makefile')
if not os.path.exists(top_make):
print('Your tree has not been configured yet. Please run '
'|mach build| with no arguments.')
return 1
# Collect target pairs.
target_pairs = []
for target in what:
path_arg = self._wrap_path_argument(target)
if directory is not None:
make_dir = os.path.join(self.topobjdir, directory)
make_target = target
else:
make_dir, make_target = \
resolve_target_to_make(self.topobjdir,
path_arg.relpath())
if make_dir is None and make_target is None:
return 1
# See bug 886162 - we don't want to "accidentally" build
# the entire tree (if that's really the intent, it's
# unlikely they would have specified a directory.)
if not make_dir and not make_target:
print("The specified directory doesn't contain a "
"Makefile and the first parent with one is the "
"root of the tree. Please specify a directory "
"with a Makefile or run |mach build| if you "
"want to build the entire tree.")
return 1
target_pairs.append((make_dir, make_target))
# Possibly add extra make depencies using dumbmake.
if not disable_extra_make_dependencies:
from dumbmake.dumbmake import (dependency_map,
add_extra_dependencies)
depfile = os.path.join(self.topsrcdir, 'build',
'dumbmake-dependencies')
with open(depfile) as f:
dm = dependency_map(f.readlines())
new_pairs = list(add_extra_dependencies(target_pairs, dm))
self.log(logging.DEBUG, 'dumbmake',
{'target_pairs': target_pairs,
'new_pairs': new_pairs},
'Added extra dependencies: will build {new_pairs} ' +
'instead of {target_pairs}.')
target_pairs = new_pairs
# Ensure build backend is up to date. The alternative is to
# have rules in the invoked Makefile to rebuild the build
# backend. But that involves make reinvoking itself and there
# are undesired side-effects of this. See bug 877308 for a
# comprehensive history lesson.
self._run_make(directory=self.topobjdir, target='backend',
line_handler=output.on_line, log=False,
print_directory=False, keep_going=keep_going)
# Build target pairs.
for make_dir, make_target in target_pairs:
# We don't display build status messages during partial
# tree builds because they aren't reliable there. This
# could potentially be fixed if the build monitor were more
# intelligent about encountering undefined state.
status = self._run_make(directory=make_dir, target=make_target,
line_handler=output.on_line, log=False, print_directory=False,
ensure_exit_code=False, num_jobs=jobs, silent=not verbose,
append_env={b'NO_BUILDSTATUS_MESSAGES': b'1'},
keep_going=keep_going)
if status != 0:
break
else:
# Try to call the default backend's build() method. This will
# run configure to determine BUILD_BACKENDS if it hasn't run
# yet.
config = None
try:
config = self.config_environment
except Exception:
config_rc = self.configure(buildstatus_messages=True,
line_handler=output.on_line)
if config_rc != 0:
return config_rc
# Even if configure runs successfully, we may have trouble
# getting the config_environment for some builds, such as
# OSX Universal builds. These have to go through client.mk
# regardless.
try:
config = self.config_environment
except Exception:
pass
if config:
active_backend = config.substs.get('BUILD_BACKENDS', [None])[0]
if active_backend:
backend_cls = get_backend_class(active_backend)(config)
status = backend_cls.build(self, output, jobs, verbose)
# If the backend doesn't specify a build() method, then just
# call client.mk directly.
if status is None:
status = self._run_client_mk(line_handler=output.on_line,
jobs=jobs,
verbose=verbose,
keep_going=keep_going)
self.log(logging.WARNING, 'warning_summary',
{'count': len(monitor.warnings_database)},
'{count} compiler warnings present.')
# Try to run the active build backend's post-build step, if possible.
try:
config = self.config_environment
active_backend = config.substs.get('BUILD_BACKENDS', [None])[0]
if active_backend:
backend_cls = get_backend_class(active_backend)(config)
new_status = backend_cls.post_build(self, output, jobs, verbose, status)
status = new_status
except Exception as ex:
self.log(logging.DEBUG, 'post_build', {'ex': ex},
"Unable to run active build backend's post-build step; " +
"failing the build due to exception: {ex}.")
if not status:
# If the underlying build provided a failing status, pass
# it through; otherwise, fail.
status = 1
monitor.finish(record_usage=status == 0)
# Print the collected compiler warnings. This is redundant with
# inline output from the compiler itself. However, unlike inline
# output, this list is sorted and grouped by file, making it
# easier to triage output.
#
# Only do this if we had a successful build. If the build failed,
# there are more important things in the log to look for than
# whatever code we warned about.
if not status:
# Suppress warnings for 3rd party projects in local builds
# until we suppress them for real.
# TODO remove entries/feature once we stop generating warnings
# in these directories.
pathToThirdparty = os.path.join(self.topsrcdir,
"tools",
"rewriting",
"ThirdPartyPaths.txt")
if os.path.exists(pathToThirdparty):
with open(pathToThirdparty) as f:
# Normalize the path (no trailing /)
LOCAL_SUPPRESS_DIRS = tuple(d.rstrip('/') for d in f.read().splitlines())
else:
# For application based on gecko like thunderbird
LOCAL_SUPPRESS_DIRS = ()
suppressed_by_dir = Counter()
for warning in sorted(monitor.instance_warnings):
path = mozpath.normsep(warning['filename'])
if path.startswith(self.topsrcdir):
path = path[len(self.topsrcdir) + 1:]
warning['normpath'] = path
if (path.startswith(LOCAL_SUPPRESS_DIRS) and
'MOZ_AUTOMATION' not in os.environ):
for d in LOCAL_SUPPRESS_DIRS:
if path.startswith(d):
suppressed_by_dir[d] += 1
break
continue
if warning['column'] is not None:
self.log(logging.WARNING, 'compiler_warning', warning,
'warning: {normpath}:{line}:{column} [{flag}] '
'{message}')
else:
self.log(logging.WARNING, 'compiler_warning', warning,
'warning: {normpath}:{line} [{flag}] {message}')
for d, count in sorted(suppressed_by_dir.items()):
self.log(logging.WARNING, 'suppressed_warning',
{'dir': d, 'count': count},
'(suppressed {count} warnings in {dir})')
high_finder, finder_percent = monitor.have_high_finder_usage()
if high_finder:
print(FINDER_SLOW_MESSAGE % finder_percent)
ccache_end = monitor.ccache_stats()
ccache_diff = None
if ccache_start and ccache_end:
ccache_diff = ccache_end - ccache_start
if ccache_diff:
self.log(logging.INFO, 'ccache',
{'msg': ccache_diff.hit_rate_message()}, "{msg}")
notify_minimum_time = 300
try:
notify_minimum_time = int(os.environ.get('MACH_NOTIFY_MINTIME', '300'))
except ValueError:
# Just stick with the default
pass
if monitor.elapsed > notify_minimum_time:
# Display a notification when the build completes.
self.notify('Build complete' if not status else 'Build failed')
if status:
return status
long_build = monitor.elapsed > 600
if long_build:
output.on_line('We know it took a while, but your build finally finished successfully!')
else:
output.on_line('Your build was successful!')
if monitor.have_resource_usage:
excessive, swap_in, swap_out = monitor.have_excessive_swapping()
# if excessive:
# print(EXCESSIVE_SWAP_MESSAGE)
print('To view resource usage of the build, run |mach '
'resource-usage|.')
telemetry_handler = getattr(mach_context,
'telemetry_handler', None)
telemetry_data = monitor.get_resource_usage()
# Record build configuration data. For now, we cherry pick
# items we need rather than grabbing everything, in order
# to avoid accidentally disclosing PII.
telemetry_data['substs'] = {}
try:
for key in ['MOZ_ARTIFACT_BUILDS', 'MOZ_USING_CCACHE', 'MOZ_USING_SCCACHE']:
value = self.substs.get(key, False)
telemetry_data['substs'][key] = value
except BuildEnvironmentNotFoundException:
pass
# Grab ccache stats if available. We need to be careful not
# to capture information that can potentially identify the
# user (such as the cache location)
if ccache_diff:
telemetry_data['ccache'] = {}
for key in [key[0] for key in ccache_diff.STATS_KEYS]:
try:
telemetry_data['ccache'][key] = ccache_diff._values[key]
except KeyError:
pass
if telemetry_handler:
telemetry_handler(mach_context, telemetry_data)
# Only for full builds because incremental builders likely don't
# need to be burdened with this.
if not what:
try:
# Fennec doesn't have useful output from just building. We should
# arguably make the build action useful for Fennec. Another day...
if self.substs['MOZ_BUILD_APP'] != 'mobile/android':
print('To take your build for a test drive, run: |mach run|')
app = self.substs['MOZ_BUILD_APP']
if app in ('browser', 'mobile/android'):
print('For more information on what to do now, see '
'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox')
except Exception:
# Ignore Exceptions in case we can't find config.status (such
# as when doing OSX Universal builds)
pass
return status
