def build(self):
# This code is only meant to be temporary until the more robust tree
# building code in bug 780329 lands.
from mozbuild.compilation.warnings import WarningsCollector
from mozbuild.compilation.warnings import WarningsDatabase
warnings_path = self._get_state_filename('warnings.json')
warnings_database = WarningsDatabase()
if os.path.exists(warnings_path):
warnings_database.load_from_file(warnings_path)
warnings_collector = WarningsCollector(database=warnings_database,
objdir=self.topobjdir)
def on_line(line):
try:
warning = warnings_collector.process_line(line)
if warning:
self.log(logging.INFO, 'compiler_warning', warning,
'Warning: {flag} in {filename}: {message}')
except:
# This will get logged in the more robust implementation.
pass
self.log(logging.INFO, 'build_output', {'line': line}, '{line}')
self._run_make(srcdir=True, filename='client.mk', line_handler=on_line,
log=False, print_directory=False)
self.log(logging.WARNING, 'warning_summary',
{'count': len(warnings_collector.database)},
'{count} compiler warnings present.')
warnings_database.save_to_file(warnings_path)
def database(self):
from mozbuild.compilation.warnings import WarningsDatabase
path = self.database_path
database = WarningsDatabase()
if os.path.exists(path):
database.load_from_file(path)
return database
def database(self):
from mozbuild.compilation.warnings import WarningsDatabase
path = self.database_path
database = WarningsDatabase()
if os.path.exists(path):
database.load_from_file(path)
return database
def test_basic(self):
db = WarningsDatabase()
self.assertEqual(len(db), 0)
for i in range(10):
db.insert(get_warning(), compute_hash=False)
self.assertEqual(len(db), 10)
warnings = list(db)
self.assertEqual(len(warnings), 10)
def test_basic(self):
db = WarningsDatabase()
self.assertEqual(len(db), 0)
for i in range(10):
db.insert(get_warning(), compute_hash=False)
self.assertEqual(len(db), 10)
warnings = list(db)
self.assertEqual(len(warnings), 10)
def test_pruning(self):
"""Ensure old warnings are removed from database appropriately."""
db = WarningsDatabase()
source_files = []
for i in range(1, 21):
temp = NamedTemporaryFile(mode='wt')
temp.write('x' * (100 * i))
temp.flush()
# Keep reference so it doesn't get GC'd and deleted.
source_files.append(temp)
w = CompilerWarning()
w['filename'] = temp.name
w['line'] = 1
w['column'] = i * 10
w['message'] = 'irrelevant'
db.insert(w)
self.assertEqual(len(db), 20)
# If we change a source file, inserting a new warning should nuke the
# old one.
source_files[0].write('extra')
source_files[0].flush()
w = CompilerWarning()
w['filename'] = source_files[0].name
w['line'] = 1
w['column'] = 50
w['message'] = 'replaced'
db.insert(w)
self.assertEqual(len(db), 20)
warnings = list(db.warnings_for_file(source_files[0].name))
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0]['column'], w['column'])
# If we delete the source file, calling prune should cause the warnings
# to go away.
old_filename = source_files[0].name
del source_files[0]
self.assertFalse(os.path.exists(old_filename))
db.prune()
self.assertEqual(len(db), 19)
def test_pruning(self):
"""Ensure old warnings are removed from database appropriately."""
db = WarningsDatabase()
source_files = []
for i in range(1, 21):
temp = NamedTemporaryFile(mode='wt')
temp.write('x' * (100 * i))
temp.flush()
# Keep reference so it doesn't get GC'd and deleted.
source_files.append(temp)
w = CompilerWarning()
w['filename'] = temp.name
w['line'] = 1
w['column'] = i * 10
w['message'] = 'irrelevant'
db.insert(w)
self.assertEqual(len(db), 20)
# If we change a source file, inserting a new warning should nuke the
# old one.
source_files[0].write('extra')
source_files[0].flush()
w = CompilerWarning()
w['filename'] = source_files[0].name
w['line'] = 1
w['column'] = 50
w['message'] = 'replaced'
db.insert(w)
self.assertEqual(len(db), 20)
warnings = list(db.warnings_for_file(source_files[0].name))
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0]['column'], w['column'])
# If we delete the source file, calling prune should cause the warnings
# to go away.
old_filename = source_files[0].name
del source_files[0]
self.assertFalse(os.path.exists(old_filename))
db.prune()
self.assertEqual(len(db), 19)
def test_hashing(self):
"""Ensure that hashing files on insert works."""
db = WarningsDatabase()
temp = NamedTemporaryFile(mode='wt')
temp.write('x' * 100)
temp.flush()
w = CompilerWarning()
w['filename'] = temp.name
w['line'] = 1
w['column'] = 4
w['message'] = 'foo bar'
# Should not throw.
db.insert(w)
w['filename'] = 'DOES_NOT_EXIST'
with self.assertRaises(Exception):
db.insert(w)
def build(self):
# This code is only meant to be temporary until the more robust tree
# building code in bug 780329 lands.
from mozbuild.compilation.warnings import WarningsCollector
from mozbuild.compilation.warnings import WarningsDatabase
warnings_path = self._get_state_filename('warnings.json')
warnings_database = WarningsDatabase()
if os.path.exists(warnings_path):
warnings_database.load_from_file(warnings_path)
warnings_collector = WarningsCollector(database=warnings_database,
objdir=self.topobjdir)
def on_line(line):
try:
warning = warnings_collector.process_line(line)
if warning:
self.log(logging.INFO, 'compiler_warning', warning,
'Warning: {flag} in {filename}: {message}')
except:
# This will get logged in the more robust implementation.
pass
self.log(logging.INFO, 'build_output', {'line': line}, '{line}')
status = self._run_make(srcdir=True,
filename='client.mk',
line_handler=on_line,
log=False,
print_directory=False,
ensure_exit_code=False)
self.log(logging.WARNING, 'warning_summary',
{'count': len(warnings_collector.database)},
'{count} compiler warnings present.')
warnings_database.prune()
warnings_database.save_to_file(warnings_path)
return status
def test_hashing(self):
"""Ensure that hashing files on insert works."""
db = WarningsDatabase()
temp = NamedTemporaryFile(mode="wt")
temp.write("x" * 100)
temp.flush()
w = CompilerWarning()
w["filename"] = temp.name
w["line"] = 1
w["column"] = 4
w["message"] = "foo bar"
# Should not throw.
db.insert(w)
w["filename"] = "DOES_NOT_EXIST"
with self.assertRaises(Exception):
db.insert(w)
def test_hashing(self):
"""Ensure that hashing files on insert works."""
db = WarningsDatabase()
temp = NamedTemporaryFile(mode='wt')
temp.write('x' * 100)
temp.flush()
w = CompilerWarning()
w['filename'] = temp.name
w['line'] = 1
w['column'] = 4
w['message'] = 'foo bar'
# Should not throw.
db.insert(w)
w['filename'] = 'DOES_NOT_EXIST'
with self.assertRaises(Exception):
db.insert(w)
def build(self, what=None):
# This code is only meant to be temporary until the more robust tree
# building code in bug 780329 lands.
from mozbuild.compilation.warnings import WarningsCollector
from mozbuild.compilation.warnings import WarningsDatabase
warnings_path = self._get_state_filename('warnings.json')
warnings_database = WarningsDatabase()
if os.path.exists(warnings_path):
try:
warnings_database.load_from_file(warnings_path)
except ValueError:
os.remove(warnings_path)
warnings_collector = WarningsCollector(database=warnings_database,
objdir=self.topobjdir)
def on_line(line):
try:
warning = warnings_collector.process_line(line)
if warning:
self.log(logging.INFO, 'compiler_warning', warning,
'Warning: {flag} in {filename}: {message}')
except:
# This will get logged in the more robust implementation.
pass
self.log(logging.INFO, 'build_output', {'line': line}, '{line}')
def resolve_target_to_make(target):
if os.path.isabs(target):
print('Absolute paths for make targets are not allowed.')
return (None, None)
target = target.replace(os.sep, '/')
abs_target = os.path.join(self.topobjdir, target)
# For directories, run |make -C dir|. If the directory does not
# contain a Makefile, check parents until we find one. At worst,
# this will terminate at the root.
if os.path.isdir(abs_target):
current = abs_target
while True:
make_path = os.path.join(current, 'Makefile')
if os.path.exists(make_path):
return (current[len(self.topobjdir) + 1:], None)
current = os.path.dirname(current)
# If it's not in a directory, this is probably a top-level make
# target. Treat it as such.
if '/' not in target:
return (None, target)
# We have a relative path within the tree. We look for a Makefile
# as far into the path as possible. Then, we compute the make
# target as relative to that directory.
reldir = os.path.dirname(target)
target = os.path.basename(target)
while True:
make_path = os.path.join(self.topobjdir, reldir, 'Makefile')
if os.path.exists(make_path):
return (reldir, target)
target = os.path.join(os.path.basename(reldir), target)
reldir = os.path.dirname(reldir)
# End of resolve_target_to_make.
if what:
top_make = os.path.join(self.topobjdir, 'Makefile')
if not os.path.exists(top_make):
print('Your tree has not been configured yet. Please run '
'|mach build| with no arguments.')
return 1
for target in what:
make_dir, make_target = resolve_target_to_make(target)
if make_dir is None and make_target is None:
return 1
status = self._run_make(directory=make_dir,
target=make_target,
line_handler=on_line,
log=False,
print_directory=False,
ensure_exit_code=False)
if status != 0:
break
else:
status = self._run_make(srcdir=True,
filename='client.mk',
line_handler=on_line,
log=False,
print_directory=False,
allow_parallel=False,
ensure_exit_code=False)
self.log(logging.WARNING, 'warning_summary',
{'count': len(warnings_collector.database)},
'{count} compiler warnings present.')
warnings_database.prune()
warnings_database.save_to_file(warnings_path)
print('Finished building. Built files are in %s' % self.topobjdir)
return status
def build(self, what=None):
# This code is only meant to be temporary until the more robust tree
# building code in bug 780329 lands.
from mozbuild.compilation.warnings import WarningsCollector
from mozbuild.compilation.warnings import WarningsDatabase
from mozbuild.util import resolve_target_to_make
warnings_path = self._get_state_filename('warnings.json')
warnings_database = WarningsDatabase()
if os.path.exists(warnings_path):
try:
warnings_database.load_from_file(warnings_path)
except ValueError:
os.remove(warnings_path)
warnings_collector = WarningsCollector(database=warnings_database,
objdir=self.topobjdir)
def on_line(line):
try:
warning = warnings_collector.process_line(line)
if warning:
self.log(logging.INFO, 'compiler_warning', warning,
'Warning: {flag} in {filename}: {message}')
except:
# This will get logged in the more robust implementation.
pass
self.log(logging.INFO, 'build_output', {'line': line}, '{line}')
finder_start_cpu = self._get_finder_cpu_usage()
time_start = time.time()
if what:
top_make = os.path.join(self.topobjdir, 'Makefile')
if not os.path.exists(top_make):
print('Your tree has not been configured yet. Please run '
'|mach build| with no arguments.')
return 1
for target in what:
path_arg = self._wrap_path_argument(target)
make_dir, make_target = resolve_target_to_make(self.topobjdir,
path_arg.relpath())
if make_dir is None and make_target is None:
return 1
status = self._run_make(directory=make_dir, target=make_target,
line_handler=on_line, log=False, print_directory=False,
ensure_exit_code=False)
if status != 0:
break
else:
status = self._run_make(srcdir=True, filename='client.mk',
line_handler=on_line, log=False, print_directory=False,
allow_parallel=False, ensure_exit_code=False)
self.log(logging.WARNING, 'warning_summary',
{'count': len(warnings_collector.database)},
'{count} compiler warnings present.')
warnings_database.prune()
warnings_database.save_to_file(warnings_path)
time_end = time.time()
time_elapsed = time_end - time_start
self._handle_finder_cpu_usage(time_elapsed, finder_start_cpu)
long_build = time_elapsed > 600
if status:
return status
if long_build:
print('We know it took a while, but your build finally finished successfully!')
else:
print('Your build was successful!')
app_path = self.get_binary_path('app')
print('To take your build for a test drive, run: %s' % app_path)
# Only for full builds because incremental builders likely don't
# need to be burdened with this.
if not what:
app = self.substs['MOZ_BUILD_APP']
if app in ('browser', 'mobile/android'):
print('For more information on what to do now, see '
'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox')
return status
def build(self, what=None):
# This code is only meant to be temporary until the more robust tree
# building code in bug 780329 lands.
from mozbuild.compilation.warnings import WarningsCollector
from mozbuild.compilation.warnings import WarningsDatabase
from mozbuild.util import resolve_target_to_make
warnings_path = self._get_state_filename('warnings.json')
warnings_database = WarningsDatabase()
if os.path.exists(warnings_path):
try:
warnings_database.load_from_file(warnings_path)
except ValueError:
os.remove(warnings_path)
warnings_collector = WarningsCollector(database=warnings_database,
objdir=self.topobjdir)
def on_line(line):
try:
warning = warnings_collector.process_line(line)
if warning:
self.log(logging.INFO, 'compiler_warning', warning,
'Warning: {flag} in {filename}: {message}')
except:
# This will get logged in the more robust implementation.
pass
self.log(logging.INFO, 'build_output', {'line': line}, '{line}')
finder_start_cpu = self._get_finder_cpu_usage()
time_start = time.time()
if what:
top_make = os.path.join(self.topobjdir, 'Makefile')
if not os.path.exists(top_make):
print('Your tree has not been configured yet. Please run '
'|mach build| with no arguments.')
return 1
for target in what:
path_arg = self._wrap_path_argument(target)
make_dir, make_target = resolve_target_to_make(
self.topobjdir, path_arg.relpath())
if make_dir is None and make_target is None:
return 1
status = self._run_make(directory=make_dir,
target=make_target,
line_handler=on_line,
log=False,
print_directory=False,
ensure_exit_code=False)
if status != 0:
break
else:
status = self._run_make(srcdir=True,
filename='client.mk',
line_handler=on_line,
log=False,
print_directory=False,
allow_parallel=False,
ensure_exit_code=False)
self.log(logging.WARNING, 'warning_summary',
{'count': len(warnings_collector.database)},
'{count} compiler warnings present.')
warnings_database.prune()
warnings_database.save_to_file(warnings_path)
time_end = time.time()
self._handle_finder_cpu_usage(time_end - time_start, finder_start_cpu)
print('Finished building. Built files are in %s' % self.topobjdir)
return status
def build(self, what=None, disable_extra_make_dependencies=None, jobs=0):
# This code is only meant to be temporary until the more robust tree
# building code in bug 780329 lands.
from mozbuild.compilation.warnings import WarningsCollector
from mozbuild.compilation.warnings import WarningsDatabase
from mozbuild.util import resolve_target_to_make
warnings_path = self._get_state_filename('warnings.json')
warnings_database = WarningsDatabase()
if os.path.exists(warnings_path):
try:
warnings_database.load_from_file(warnings_path)
except ValueError:
os.remove(warnings_path)
warnings_collector = WarningsCollector(database=warnings_database,
objdir=self.topobjdir)
def on_line(line):
try:
warning = warnings_collector.process_line(line)
if warning:
self.log(logging.INFO, 'compiler_warning', warning,
'Warning: {flag} in {filename}: {message}')
except:
# This will get logged in the more robust implementation.
pass
self.log(logging.INFO, 'build_output', {'line': line}, '{line}')
finder_start_cpu = self._get_finder_cpu_usage()
time_start = time.time()
if what:
top_make = os.path.join(self.topobjdir, 'Makefile')
if not os.path.exists(top_make):
print('Your tree has not been configured yet. Please run '
'|mach build| with no arguments.')
return 1
# Collect target pairs.
target_pairs = []
for target in what:
path_arg = self._wrap_path_argument(target)
make_dir, make_target = resolve_target_to_make(self.topobjdir,
path_arg.relpath())
if make_dir is None and make_target is None:
return 1
target_pairs.append((make_dir, make_target))
# Possibly add extra make depencies using dumbmake.
if not disable_extra_make_dependencies:
from dumbmake.dumbmake import (dependency_map,
add_extra_dependencies)
depfile = os.path.join(self.topsrcdir, 'build',
'dumbmake-dependencies')
with open(depfile) as f:
dm = dependency_map(f.readlines())
new_pairs = list(add_extra_dependencies(target_pairs, dm))
self.log(logging.DEBUG, 'dumbmake',
{'target_pairs': target_pairs,
'new_pairs': new_pairs},
'Added extra dependencies: will build {new_pairs} ' +
'instead of {target_pairs}.')
target_pairs = new_pairs
# Build target pairs.
for make_dir, make_target in target_pairs:
status = self._run_make(directory=make_dir, target=make_target,
line_handler=on_line, log=False, print_directory=False,
ensure_exit_code=False, num_jobs=jobs)
if status != 0:
break
else:
status = self._run_make(srcdir=True, filename='client.mk',
line_handler=on_line, log=False, print_directory=False,
allow_parallel=False, ensure_exit_code=False, num_jobs=jobs)
self.log(logging.WARNING, 'warning_summary',
{'count': len(warnings_collector.database)},
'{count} compiler warnings present.')
warnings_database.prune()
warnings_database.save_to_file(warnings_path)
time_end = time.time()
time_elapsed = time_end - time_start
self._handle_finder_cpu_usage(time_elapsed, finder_start_cpu)
long_build = time_elapsed > 600
if status:
return status
if long_build:
print('We know it took a while, but your build finally finished successfully!')
else:
print('Your build was successful!')
# Only for full builds because incremental builders likely don't
# need to be burdened with this.
if not what:
# Fennec doesn't have useful output from just building. We should
# arguably make the build action useful for Fennec. Another day...
if self.substs['MOZ_BUILD_APP'] != 'mobile/android':
app_path = self.get_binary_path('app')
print('To take your build for a test drive, run: %s' % app_path)
app = self.substs['MOZ_BUILD_APP']
if app in ('browser', 'mobile/android'):
print('For more information on what to do now, see '
'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox')
return status
def build(self, what=None):
# This code is only meant to be temporary until the more robust tree
# building code in bug 780329 lands.
from mozbuild.compilation.warnings import WarningsCollector
from mozbuild.compilation.warnings import WarningsDatabase
warnings_path = self._get_state_filename('warnings.json')
warnings_database = WarningsDatabase()
if os.path.exists(warnings_path):
try:
warnings_database.load_from_file(warnings_path)
except ValueError:
os.remove(warnings_path)
warnings_collector = WarningsCollector(database=warnings_database,
objdir=self.topobjdir)
def on_line(line):
try:
warning = warnings_collector.process_line(line)
if warning:
self.log(logging.INFO, 'compiler_warning', warning,
'Warning: {flag} in {filename}: {message}')
except:
# This will get logged in the more robust implementation.
pass
self.log(logging.INFO, 'build_output', {'line': line}, '{line}')
def resolve_target_to_make(target):
if os.path.isabs(target):
print('Absolute paths for make targets are not allowed.')
return (None, None)
target = target.replace(os.sep, '/')
abs_target = os.path.join(self.topobjdir, target)
# For directories, run |make -C dir|. If the directory does not
# contain a Makefile, check parents until we find one. At worst,
# this will terminate at the root.
if os.path.isdir(abs_target):
current = abs_target
while True:
make_path = os.path.join(current, 'Makefile')
if os.path.exists(make_path):
return (current[len(self.topobjdir) + 1:], None)
current = os.path.dirname(current)
# If it's not in a directory, this is probably a top-level make
# target. Treat it as such.
if '/' not in target:
return (None, target)
# We have a relative path within the tree. We look for a Makefile
# as far into the path as possible. Then, we compute the make
# target as relative to that directory.
reldir = os.path.dirname(target)
target = os.path.basename(target)
while True:
make_path = os.path.join(self.topobjdir, reldir, 'Makefile')
if os.path.exists(make_path):
return (reldir, target)
target = os.path.join(os.path.basename(reldir), target)
reldir = os.path.dirname(reldir)
# End of resolve_target_to_make.
if what:
top_make = os.path.join(self.topobjdir, 'Makefile')
if not os.path.exists(top_make):
print('Your tree has not been configured yet. Please run '
'|mach build| with no arguments.')
return 1
for target in what:
make_dir, make_target = resolve_target_to_make(target)
if make_dir is None and make_target is None:
return 1
status = self._run_make(directory=make_dir, target=make_target,
line_handler=on_line, log=False, print_directory=False,
ensure_exit_code=False)
if status != 0:
break
else:
status = self._run_make(srcdir=True, filename='client.mk',
line_handler=on_line, log=False, print_directory=False,
allow_parallel=False, ensure_exit_code=False)
self.log(logging.WARNING, 'warning_summary',
{'count': len(warnings_collector.database)},
'{count} compiler warnings present.')
warnings_database.prune()
warnings_database.save_to_file(warnings_path)
print('Finished building. Built files are in %s' % self.topobjdir)
return status
def build(self, what=None):
# This code is only meant to be temporary until the more robust tree
# building code in bug 780329 lands.
from mozbuild.compilation.warnings import WarningsCollector
from mozbuild.compilation.warnings import WarningsDatabase
from mozbuild.util import resolve_target_to_make
warnings_path = self._get_state_filename('warnings.json')
warnings_database = WarningsDatabase()
if os.path.exists(warnings_path):
try:
warnings_database.load_from_file(warnings_path)
except ValueError:
os.remove(warnings_path)
warnings_collector = WarningsCollector(database=warnings_database,
objdir=self.topobjdir)
def on_line(line):
try:
warning = warnings_collector.process_line(line)
if warning:
self.log(logging.INFO, 'compiler_warning', warning,
'Warning: {flag} in {filename}: {message}')
except:
# This will get logged in the more robust implementation.
pass
self.log(logging.INFO, 'build_output', {'line': line}, '{line}')
finder_start_cpu = self._get_finder_cpu_usage()
time_start = time.time()
if what:
top_make = os.path.join(self.topobjdir, 'Makefile')
if not os.path.exists(top_make):
print('Your tree has not been configured yet. Please run '
'|mach build| with no arguments.')
return 1
for target in what:
path_arg = self._wrap_path_argument(target)
make_dir, make_target = resolve_target_to_make(self.topobjdir,
path_arg.relpath())
if make_dir is None and make_target is None:
return 1
status = self._run_make(directory=make_dir, target=make_target,
line_handler=on_line, log=False, print_directory=False,
ensure_exit_code=False)
if status != 0:
break
else:
status = self._run_make(srcdir=True, filename='client.mk',
line_handler=on_line, log=False, print_directory=False,
allow_parallel=False, ensure_exit_code=False)
self.log(logging.WARNING, 'warning_summary',
{'count': len(warnings_collector.database)},
'{count} compiler warnings present.')
warnings_database.prune()
warnings_database.save_to_file(warnings_path)
time_end = time.time()
self._handle_finder_cpu_usage(time_end - time_start, finder_start_cpu)
print('Finished building. Built files are in %s' % self.topobjdir)
return status
