def build(self): # This code is only meant to be temporary until the more robust tree # building code in bug 780329 lands. from mozbuild.compilation.warnings import WarningsCollector from mozbuild.compilation.warnings import WarningsDatabase warnings_path = self._get_state_filename('warnings.json') warnings_database = WarningsDatabase() if os.path.exists(warnings_path): warnings_database.load_from_file(warnings_path) warnings_collector = WarningsCollector(database=warnings_database, objdir=self.topobjdir) def on_line(line): try: warning = warnings_collector.process_line(line) if warning: self.log(logging.INFO, 'compiler_warning', warning, 'Warning: {flag} in {filename}: {message}') except: # This will get logged in the more robust implementation. pass self.log(logging.INFO, 'build_output', {'line': line}, '{line}') self._run_make(srcdir=True, filename='client.mk', line_handler=on_line, log=False, print_directory=False) self.log(logging.WARNING, 'warning_summary', {'count': len(warnings_collector.database)}, '{count} compiler warnings present.') warnings_database.save_to_file(warnings_path)
def database(self): from mozbuild.compilation.warnings import WarningsDatabase path = self.database_path database = WarningsDatabase() if os.path.exists(path): database.load_from_file(path) return database
def test_basic(self): db = WarningsDatabase() self.assertEqual(len(db), 0) for i in range(10): db.insert(get_warning(), compute_hash=False) self.assertEqual(len(db), 10) warnings = list(db) self.assertEqual(len(warnings), 10)
def test_pruning(self): """Ensure old warnings are removed from database appropriately.""" db = WarningsDatabase() source_files = [] for i in range(1, 21): temp = NamedTemporaryFile(mode='wt') temp.write('x' * (100 * i)) temp.flush() # Keep reference so it doesn't get GC'd and deleted. source_files.append(temp) w = CompilerWarning() w['filename'] = temp.name w['line'] = 1 w['column'] = i * 10 w['message'] = 'irrelevant' db.insert(w) self.assertEqual(len(db), 20) # If we change a source file, inserting a new warning should nuke the # old one. source_files[0].write('extra') source_files[0].flush() w = CompilerWarning() w['filename'] = source_files[0].name w['line'] = 1 w['column'] = 50 w['message'] = 'replaced' db.insert(w) self.assertEqual(len(db), 20) warnings = list(db.warnings_for_file(source_files[0].name)) self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0]['column'], w['column']) # If we delete the source file, calling prune should cause the warnings # to go away. old_filename = source_files[0].name del source_files[0] self.assertFalse(os.path.exists(old_filename)) db.prune() self.assertEqual(len(db), 19)
def test_hashing(self): """Ensure that hashing files on insert works.""" db = WarningsDatabase() temp = NamedTemporaryFile(mode='wt') temp.write('x' * 100) temp.flush() w = CompilerWarning() w['filename'] = temp.name w['line'] = 1 w['column'] = 4 w['message'] = 'foo bar' # Should not throw. db.insert(w) w['filename'] = 'DOES_NOT_EXIST' with self.assertRaises(Exception): db.insert(w)
def build(self): # This code is only meant to be temporary until the more robust tree # building code in bug 780329 lands. from mozbuild.compilation.warnings import WarningsCollector from mozbuild.compilation.warnings import WarningsDatabase warnings_path = self._get_state_filename('warnings.json') warnings_database = WarningsDatabase() if os.path.exists(warnings_path): warnings_database.load_from_file(warnings_path) warnings_collector = WarningsCollector(database=warnings_database, objdir=self.topobjdir) def on_line(line): try: warning = warnings_collector.process_line(line) if warning: self.log(logging.INFO, 'compiler_warning', warning, 'Warning: {flag} in {filename}: {message}') except: # This will get logged in the more robust implementation. pass self.log(logging.INFO, 'build_output', {'line': line}, '{line}') status = self._run_make(srcdir=True, filename='client.mk', line_handler=on_line, log=False, print_directory=False, ensure_exit_code=False) self.log(logging.WARNING, 'warning_summary', {'count': len(warnings_collector.database)}, '{count} compiler warnings present.') warnings_database.prune() warnings_database.save_to_file(warnings_path) return status
def test_hashing(self): """Ensure that hashing files on insert works.""" db = WarningsDatabase() temp = NamedTemporaryFile(mode="wt") temp.write("x" * 100) temp.flush() w = CompilerWarning() w["filename"] = temp.name w["line"] = 1 w["column"] = 4 w["message"] = "foo bar" # Should not throw. db.insert(w) w["filename"] = "DOES_NOT_EXIST" with self.assertRaises(Exception): db.insert(w)
def build(self, what=None): # This code is only meant to be temporary until the more robust tree # building code in bug 780329 lands. from mozbuild.compilation.warnings import WarningsCollector from mozbuild.compilation.warnings import WarningsDatabase warnings_path = self._get_state_filename('warnings.json') warnings_database = WarningsDatabase() if os.path.exists(warnings_path): try: warnings_database.load_from_file(warnings_path) except ValueError: os.remove(warnings_path) warnings_collector = WarningsCollector(database=warnings_database, objdir=self.topobjdir) def on_line(line): try: warning = warnings_collector.process_line(line) if warning: self.log(logging.INFO, 'compiler_warning', warning, 'Warning: {flag} in {filename}: {message}') except: # This will get logged in the more robust implementation. pass self.log(logging.INFO, 'build_output', {'line': line}, '{line}') def resolve_target_to_make(target): if os.path.isabs(target): print('Absolute paths for make targets are not allowed.') return (None, None) target = target.replace(os.sep, '/') abs_target = os.path.join(self.topobjdir, target) # For directories, run |make -C dir|. If the directory does not # contain a Makefile, check parents until we find one. At worst, # this will terminate at the root. if os.path.isdir(abs_target): current = abs_target while True: make_path = os.path.join(current, 'Makefile') if os.path.exists(make_path): return (current[len(self.topobjdir) + 1:], None) current = os.path.dirname(current) # If it's not in a directory, this is probably a top-level make # target. Treat it as such. if '/' not in target: return (None, target) # We have a relative path within the tree. We look for a Makefile # as far into the path as possible. Then, we compute the make # target as relative to that directory. reldir = os.path.dirname(target) target = os.path.basename(target) while True: make_path = os.path.join(self.topobjdir, reldir, 'Makefile') if os.path.exists(make_path): return (reldir, target) target = os.path.join(os.path.basename(reldir), target) reldir = os.path.dirname(reldir) # End of resolve_target_to_make. if what: top_make = os.path.join(self.topobjdir, 'Makefile') if not os.path.exists(top_make): print('Your tree has not been configured yet. Please run ' '|mach build| with no arguments.') return 1 for target in what: make_dir, make_target = resolve_target_to_make(target) if make_dir is None and make_target is None: return 1 status = self._run_make(directory=make_dir, target=make_target, line_handler=on_line, log=False, print_directory=False, ensure_exit_code=False) if status != 0: break else: status = self._run_make(srcdir=True, filename='client.mk', line_handler=on_line, log=False, print_directory=False, allow_parallel=False, ensure_exit_code=False) self.log(logging.WARNING, 'warning_summary', {'count': len(warnings_collector.database)}, '{count} compiler warnings present.') warnings_database.prune() warnings_database.save_to_file(warnings_path) print('Finished building. Built files are in %s' % self.topobjdir) return status
def build(self, what=None): # This code is only meant to be temporary until the more robust tree # building code in bug 780329 lands. from mozbuild.compilation.warnings import WarningsCollector from mozbuild.compilation.warnings import WarningsDatabase from mozbuild.util import resolve_target_to_make warnings_path = self._get_state_filename('warnings.json') warnings_database = WarningsDatabase() if os.path.exists(warnings_path): try: warnings_database.load_from_file(warnings_path) except ValueError: os.remove(warnings_path) warnings_collector = WarningsCollector(database=warnings_database, objdir=self.topobjdir) def on_line(line): try: warning = warnings_collector.process_line(line) if warning: self.log(logging.INFO, 'compiler_warning', warning, 'Warning: {flag} in {filename}: {message}') except: # This will get logged in the more robust implementation. pass self.log(logging.INFO, 'build_output', {'line': line}, '{line}') finder_start_cpu = self._get_finder_cpu_usage() time_start = time.time() if what: top_make = os.path.join(self.topobjdir, 'Makefile') if not os.path.exists(top_make): print('Your tree has not been configured yet. Please run ' '|mach build| with no arguments.') return 1 for target in what: path_arg = self._wrap_path_argument(target) make_dir, make_target = resolve_target_to_make(self.topobjdir, path_arg.relpath()) if make_dir is None and make_target is None: return 1 status = self._run_make(directory=make_dir, target=make_target, line_handler=on_line, log=False, print_directory=False, ensure_exit_code=False) if status != 0: break else: status = self._run_make(srcdir=True, filename='client.mk', line_handler=on_line, log=False, print_directory=False, allow_parallel=False, ensure_exit_code=False) self.log(logging.WARNING, 'warning_summary', {'count': len(warnings_collector.database)}, '{count} compiler warnings present.') warnings_database.prune() warnings_database.save_to_file(warnings_path) time_end = time.time() time_elapsed = time_end - time_start self._handle_finder_cpu_usage(time_elapsed, finder_start_cpu) long_build = time_elapsed > 600 if status: return status if long_build: print('We know it took a while, but your build finally finished successfully!') else: print('Your build was successful!') app_path = self.get_binary_path('app') print('To take your build for a test drive, run: %s' % app_path) # Only for full builds because incremental builders likely don't # need to be burdened with this. if not what: app = self.substs['MOZ_BUILD_APP'] if app in ('browser', 'mobile/android'): print('For more information on what to do now, see ' 'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox') return status
def build(self, what=None): # This code is only meant to be temporary until the more robust tree # building code in bug 780329 lands. from mozbuild.compilation.warnings import WarningsCollector from mozbuild.compilation.warnings import WarningsDatabase from mozbuild.util import resolve_target_to_make warnings_path = self._get_state_filename('warnings.json') warnings_database = WarningsDatabase() if os.path.exists(warnings_path): try: warnings_database.load_from_file(warnings_path) except ValueError: os.remove(warnings_path) warnings_collector = WarningsCollector(database=warnings_database, objdir=self.topobjdir) def on_line(line): try: warning = warnings_collector.process_line(line) if warning: self.log(logging.INFO, 'compiler_warning', warning, 'Warning: {flag} in {filename}: {message}') except: # This will get logged in the more robust implementation. pass self.log(logging.INFO, 'build_output', {'line': line}, '{line}') finder_start_cpu = self._get_finder_cpu_usage() time_start = time.time() if what: top_make = os.path.join(self.topobjdir, 'Makefile') if not os.path.exists(top_make): print('Your tree has not been configured yet. Please run ' '|mach build| with no arguments.') return 1 for target in what: path_arg = self._wrap_path_argument(target) make_dir, make_target = resolve_target_to_make( self.topobjdir, path_arg.relpath()) if make_dir is None and make_target is None: return 1 status = self._run_make(directory=make_dir, target=make_target, line_handler=on_line, log=False, print_directory=False, ensure_exit_code=False) if status != 0: break else: status = self._run_make(srcdir=True, filename='client.mk', line_handler=on_line, log=False, print_directory=False, allow_parallel=False, ensure_exit_code=False) self.log(logging.WARNING, 'warning_summary', {'count': len(warnings_collector.database)}, '{count} compiler warnings present.') warnings_database.prune() warnings_database.save_to_file(warnings_path) time_end = time.time() self._handle_finder_cpu_usage(time_end - time_start, finder_start_cpu) print('Finished building. Built files are in %s' % self.topobjdir) return status
def build(self, what=None, disable_extra_make_dependencies=None, jobs=0): # This code is only meant to be temporary until the more robust tree # building code in bug 780329 lands. from mozbuild.compilation.warnings import WarningsCollector from mozbuild.compilation.warnings import WarningsDatabase from mozbuild.util import resolve_target_to_make warnings_path = self._get_state_filename('warnings.json') warnings_database = WarningsDatabase() if os.path.exists(warnings_path): try: warnings_database.load_from_file(warnings_path) except ValueError: os.remove(warnings_path) warnings_collector = WarningsCollector(database=warnings_database, objdir=self.topobjdir) def on_line(line): try: warning = warnings_collector.process_line(line) if warning: self.log(logging.INFO, 'compiler_warning', warning, 'Warning: {flag} in {filename}: {message}') except: # This will get logged in the more robust implementation. pass self.log(logging.INFO, 'build_output', {'line': line}, '{line}') finder_start_cpu = self._get_finder_cpu_usage() time_start = time.time() if what: top_make = os.path.join(self.topobjdir, 'Makefile') if not os.path.exists(top_make): print('Your tree has not been configured yet. Please run ' '|mach build| with no arguments.') return 1 # Collect target pairs. target_pairs = [] for target in what: path_arg = self._wrap_path_argument(target) make_dir, make_target = resolve_target_to_make(self.topobjdir, path_arg.relpath()) if make_dir is None and make_target is None: return 1 target_pairs.append((make_dir, make_target)) # Possibly add extra make depencies using dumbmake. if not disable_extra_make_dependencies: from dumbmake.dumbmake import (dependency_map, add_extra_dependencies) depfile = os.path.join(self.topsrcdir, 'build', 'dumbmake-dependencies') with open(depfile) as f: dm = dependency_map(f.readlines()) new_pairs = list(add_extra_dependencies(target_pairs, dm)) self.log(logging.DEBUG, 'dumbmake', {'target_pairs': target_pairs, 'new_pairs': new_pairs}, 'Added extra dependencies: will build {new_pairs} ' + 'instead of {target_pairs}.') target_pairs = new_pairs # Build target pairs. for make_dir, make_target in target_pairs: status = self._run_make(directory=make_dir, target=make_target, line_handler=on_line, log=False, print_directory=False, ensure_exit_code=False, num_jobs=jobs) if status != 0: break else: status = self._run_make(srcdir=True, filename='client.mk', line_handler=on_line, log=False, print_directory=False, allow_parallel=False, ensure_exit_code=False, num_jobs=jobs) self.log(logging.WARNING, 'warning_summary', {'count': len(warnings_collector.database)}, '{count} compiler warnings present.') warnings_database.prune() warnings_database.save_to_file(warnings_path) time_end = time.time() time_elapsed = time_end - time_start self._handle_finder_cpu_usage(time_elapsed, finder_start_cpu) long_build = time_elapsed > 600 if status: return status if long_build: print('We know it took a while, but your build finally finished successfully!') else: print('Your build was successful!') # Only for full builds because incremental builders likely don't # need to be burdened with this. if not what: # Fennec doesn't have useful output from just building. We should # arguably make the build action useful for Fennec. Another day... if self.substs['MOZ_BUILD_APP'] != 'mobile/android': app_path = self.get_binary_path('app') print('To take your build for a test drive, run: %s' % app_path) app = self.substs['MOZ_BUILD_APP'] if app in ('browser', 'mobile/android'): print('For more information on what to do now, see ' 'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox') return status
def build(self, what=None): # This code is only meant to be temporary until the more robust tree # building code in bug 780329 lands. from mozbuild.compilation.warnings import WarningsCollector from mozbuild.compilation.warnings import WarningsDatabase from mozbuild.util import resolve_target_to_make warnings_path = self._get_state_filename('warnings.json') warnings_database = WarningsDatabase() if os.path.exists(warnings_path): try: warnings_database.load_from_file(warnings_path) except ValueError: os.remove(warnings_path) warnings_collector = WarningsCollector(database=warnings_database, objdir=self.topobjdir) def on_line(line): try: warning = warnings_collector.process_line(line) if warning: self.log(logging.INFO, 'compiler_warning', warning, 'Warning: {flag} in {filename}: {message}') except: # This will get logged in the more robust implementation. pass self.log(logging.INFO, 'build_output', {'line': line}, '{line}') finder_start_cpu = self._get_finder_cpu_usage() time_start = time.time() if what: top_make = os.path.join(self.topobjdir, 'Makefile') if not os.path.exists(top_make): print('Your tree has not been configured yet. Please run ' '|mach build| with no arguments.') return 1 for target in what: path_arg = self._wrap_path_argument(target) make_dir, make_target = resolve_target_to_make(self.topobjdir, path_arg.relpath()) if make_dir is None and make_target is None: return 1 status = self._run_make(directory=make_dir, target=make_target, line_handler=on_line, log=False, print_directory=False, ensure_exit_code=False) if status != 0: break else: status = self._run_make(srcdir=True, filename='client.mk', line_handler=on_line, log=False, print_directory=False, allow_parallel=False, ensure_exit_code=False) self.log(logging.WARNING, 'warning_summary', {'count': len(warnings_collector.database)}, '{count} compiler warnings present.') warnings_database.prune() warnings_database.save_to_file(warnings_path) time_end = time.time() self._handle_finder_cpu_usage(time_end - time_start, finder_start_cpu) print('Finished building. Built files are in %s' % self.topobjdir) return status