def test_aggregate_cpu(self): monitor = SystemResourceMonitor(poll_interval=0.25) monitor.start() time.sleep(1) monitor.stop() values = monitor.aggregate_cpu_percent() self.assertIsInstance(values, list) self.assertEqual(len(values), multiprocessing.cpu_count()) for v in values: self.assertIsInstance(v, float) value = monitor.aggregate_cpu_percent(per_cpu=False) self.assertIsInstance(value, float) values = monitor.aggregate_cpu_times() self.assertIsInstance(values, list) self.assertGreater(len(values), 0) self.assertTrue(hasattr(values[0], 'user')) t = type(values[0]) value = monitor.aggregate_cpu_times(per_cpu=False) self.assertIsInstance(value, t)
class BuildMonitor(MozbuildObject): """Monitors the output of the build.""" def init(self, warnings_path): """Create a new monitor. warnings_path is a path of a warnings database to use. """ self._warnings_path = warnings_path self.resources = SystemResourceMonitor(poll_interval=1.0) self._resources_started = False self.tiers = TierStatus(self.resources) self.warnings_database = WarningsDatabase() if os.path.exists(warnings_path): try: self.warnings_database.load_from_file(warnings_path) except ValueError: os.remove(warnings_path) self._warnings_collector = WarningsCollector( database=self.warnings_database, objdir=self.topobjdir) def start(self): """Record the start of the build.""" self.start_time = time.time() self._finder_start_cpu = self._get_finder_cpu_usage() def start_resource_recording(self): # This should be merged into start() once bug 892342 lands. self.resources.start() self._resources_started = True def on_line(self, line): """Consume a line of output from the build system. This will parse the line for state and determine whether more action is needed. Returns a BuildOutputResult instance. In this named tuple, warning will be an object describing a new parsed warning. Otherwise it will be None. state_changed indicates whether the build system changed state with this line. If the build system changed state, the caller may want to query this instance for the current state in order to update UI, etc. for_display is a boolean indicating whether the line is relevant to the user. This is typically used to filter whether the line should be presented to the user. """ if line.startswith('BUILDSTATUS'): args = line.split()[1:] action = args.pop(0) update_needed = True if action == 'TIERS': self.tiers.set_tiers(args) update_needed = False elif action == 'TIER_START': tier = args[0] self.tiers.begin_tier(tier) elif action == 'TIER_FINISH': tier, = args self.tiers.finish_tier(tier) else: raise Exception('Unknown build status: %s' % action) return BuildOutputResult(None, update_needed, False) warning = None try: warning = self._warnings_collector.process_line(line) except: pass return BuildOutputResult(warning, False, True) def finish(self, record_usage=True): """Record the end of the build.""" self.end_time = time.time() if self._resources_started: self.resources.stop() self._finder_end_cpu = self._get_finder_cpu_usage() self.elapsed = self.end_time - self.start_time self.warnings_database.prune() self.warnings_database.save_to_file(self._warnings_path) if not record_usage: return try: usage = self.record_resource_usage() if not usage: return with open(self._get_state_filename('build_resources.json'), 'w') as fh: json.dump(usage, fh, indent=2) except Exception as e: self.log(logging.WARNING, 'build_resources_error', {'msg': str(e)}, 'Exception when writing resource usage file: {msg}') def _get_finder_cpu_usage(self): """Obtain the CPU usage of the Finder app on OS X. This is used to detect high CPU usage. """ if not sys.platform.startswith('darwin'): return None if not psutil: return None for proc in psutil.process_iter(): if proc.name != 'Finder': continue if proc.username != getpass.getuser(): continue # Try to isolate system finder as opposed to other "Finder" # processes. if not proc.exe.endswith('CoreServices/Finder.app/Contents/MacOS/Finder'): continue return proc.get_cpu_times() return None def have_high_finder_usage(self): """Determine whether there was high Finder CPU usage during the build. Returns True if there was high Finder CPU usage, False if there wasn't, or None if there is nothing to report. """ if not self._finder_start_cpu: return None, None # We only measure if the measured range is sufficiently long. if self.elapsed < 15: return None, None if not self._finder_end_cpu: return None, None start = self._finder_start_cpu end = self._finder_end_cpu start_total = start.user + start.system end_total = end.user + end.system cpu_seconds = end_total - start_total # If Finder used more than 25% of 1 core during the build, report an # error. finder_percent = cpu_seconds / self.elapsed * 100 return finder_percent > 25, finder_percent def have_excessive_swapping(self): """Determine whether there was excessive swapping during the build. Returns a tuple of (excessive, swap_in, swap_out). All values are None if no swap information is available. """ if not self.have_resource_usage: return None, None, None swap_in = sum(m.swap.sin for m in self.resources.measurements) swap_out = sum(m.swap.sout for m in self.resources.measurements) # The threshold of 1024 MB has been arbitrarily chosen. # # Choosing a proper value that is ideal for everyone is hard. We will # likely iterate on the logic until people are generally satisfied. # If a value is too low, the eventual warning produced does not carry # much meaning. If the threshold is too high, people may not see the # warning and the warning will thus be ineffective. excessive = swap_in > 512 * 1048576 or swap_out > 512 * 1048576 return excessive, swap_in, swap_out @property def have_resource_usage(self): """Whether resource usage is available.""" return self.resources.start_time is not None def record_resource_usage(self): """Record the resource usage of this build. We write a log message containing a high-level summary. We also produce a data structure containing the low-level resource usage information. This data structure can e.g. be serialized into JSON and saved for subsequent analysis. If no resource usage is available, None is returned. """ if not self.have_resource_usage: return None cpu_percent = self.resources.aggregate_cpu_percent(phase=None, per_cpu=False) cpu_times = self.resources.aggregate_cpu_times(phase=None, per_cpu=False) io = self.resources.aggregate_io(phase=None) self._log_resource_usage('Overall system resources', 'resource_usage', self.end_time - self.start_time, cpu_percent, cpu_times, io) excessive, sin, sout = self.have_excessive_swapping() if excessive is not None and (sin or sout): sin /= 1048576 sout /= 1048576 self.log(logging.WARNING, 'swap_activity', {'sin': sin, 'sout': sout}, 'Swap in/out (MB): {sin}/{sout}') o = dict( version=1, start=self.start_time, end=self.end_time, duration=self.end_time - self.start_time, resources=[], cpu_percent=cpu_percent, cpu_times=cpu_times, io=io, ) o['tiers'] = self.tiers.tiered_resource_usage() self.tiers.add_resource_fields_to_dict(o) for usage in self.resources.range_usage(): cpu_percent = self.resources.aggregate_cpu_percent(usage.start, usage.end, per_cpu=False) cpu_times = self.resources.aggregate_cpu_times(usage.start, usage.end, per_cpu=False) entry = dict( start=usage.start, end=usage.end, virt=list(usage.virt), swap=list(usage.swap), ) self.tiers.add_resources_to_dict(entry, start=usage.start, end=usage.end) o['resources'].append(entry) return o def _log_resource_usage(self, prefix, m_type, duration, cpu_percent, cpu_times, io, extra_params={}): params = dict( duration=duration, cpu_percent=cpu_percent, io_reads=io.read_count, io_writes=io.write_count, io_read_bytes=io.read_bytes, io_write_bytes=io.write_bytes, io_read_time=io.read_time, io_write_time=io.write_time, ) params.update(extra_params) message = prefix + ' - Wall time: {duration:.0f}s; ' \ 'CPU: {cpu_percent:.0f}%; ' \ 'Read bytes: {io_read_bytes}; Write bytes: {io_write_bytes}; ' \ 'Read time: {io_read_time}; Write time: {io_write_time}' self.log(logging.WARNING, m_type, params, message)
class BuildMonitor(MozbuildObject): """Monitors the output of the build.""" def init(self, warnings_path): """Create a new monitor. warnings_path is a path of a warnings database to use. """ self._warnings_path = warnings_path self.resources = SystemResourceMonitor(poll_interval=1.0) self._resources_started = False self.tiers = TierStatus(self.resources) self.warnings_database = WarningsDatabase() if os.path.exists(warnings_path): try: self.warnings_database.load_from_file(warnings_path) except ValueError: os.remove(warnings_path) self._warnings_collector = WarningsCollector( database=self.warnings_database, objdir=self.topobjdir) self.build_objects = [] def start(self): """Record the start of the build.""" self.start_time = time.time() self._finder_start_cpu = self._get_finder_cpu_usage() def start_resource_recording(self): # This should be merged into start() once bug 892342 lands. self.resources.start() self._resources_started = True def on_line(self, line): """Consume a line of output from the build system. This will parse the line for state and determine whether more action is needed. Returns a BuildOutputResult instance. In this named tuple, warning will be an object describing a new parsed warning. Otherwise it will be None. state_changed indicates whether the build system changed state with this line. If the build system changed state, the caller may want to query this instance for the current state in order to update UI, etc. for_display is a boolean indicating whether the line is relevant to the user. This is typically used to filter whether the line should be presented to the user. """ if line.startswith('BUILDSTATUS'): args = line.split()[1:] action = args.pop(0) update_needed = True if action == 'TIERS': self.tiers.set_tiers(args) update_needed = False elif action == 'TIER_START': tier = args[0] self.tiers.begin_tier(tier) elif action == 'TIER_FINISH': tier, = args self.tiers.finish_tier(tier) elif action == 'OBJECT_FILE': self.build_objects.append(args[0]) update_needed = False else: raise Exception('Unknown build status: %s' % action) return BuildOutputResult(None, update_needed, False) warning = None try: warning = self._warnings_collector.process_line(line) except: pass return BuildOutputResult(warning, False, True) def stop_resource_recording(self): if self._resources_started: self.resources.stop() self._resources_started = False def finish(self, record_usage=True): """Record the end of the build.""" self.stop_resource_recording() self.end_time = time.time() self._finder_end_cpu = self._get_finder_cpu_usage() self.elapsed = self.end_time - self.start_time self.warnings_database.prune() self.warnings_database.save_to_file(self._warnings_path) if not record_usage: return try: usage = self.get_resource_usage() if not usage: return self.log_resource_usage(usage) with open(self._get_state_filename('build_resources.json'), 'w') as fh: json.dump(self.resources.as_dict(), fh, indent=2) except Exception as e: self.log(logging.WARNING, 'build_resources_error', {'msg': str(e)}, 'Exception when writing resource usage file: {msg}') def _get_finder_cpu_usage(self): """Obtain the CPU usage of the Finder app on OS X. This is used to detect high CPU usage. """ if not sys.platform.startswith('darwin'): return None if not psutil: return None for proc in psutil.process_iter(): if proc.name != 'Finder': continue if proc.username != getpass.getuser(): continue # Try to isolate system finder as opposed to other "Finder" # processes. if not proc.exe.endswith( 'CoreServices/Finder.app/Contents/MacOS/Finder'): continue return proc.get_cpu_times() return None def have_high_finder_usage(self): """Determine whether there was high Finder CPU usage during the build. Returns True if there was high Finder CPU usage, False if there wasn't, or None if there is nothing to report. """ if not self._finder_start_cpu: return None, None # We only measure if the measured range is sufficiently long. if self.elapsed < 15: return None, None if not self._finder_end_cpu: return None, None start = self._finder_start_cpu end = self._finder_end_cpu start_total = start.user + start.system end_total = end.user + end.system cpu_seconds = end_total - start_total # If Finder used more than 25% of 1 core during the build, report an # error. finder_percent = cpu_seconds / self.elapsed * 100 return finder_percent > 25, finder_percent def have_excessive_swapping(self): """Determine whether there was excessive swapping during the build. Returns a tuple of (excessive, swap_in, swap_out). All values are None if no swap information is available. """ if not self.have_resource_usage: return None, None, None swap_in = sum(m.swap.sin for m in self.resources.measurements) swap_out = sum(m.swap.sout for m in self.resources.measurements) # The threshold of 1024 MB has been arbitrarily chosen. # # Choosing a proper value that is ideal for everyone is hard. We will # likely iterate on the logic until people are generally satisfied. # If a value is too low, the eventual warning produced does not carry # much meaning. If the threshold is too high, people may not see the # warning and the warning will thus be ineffective. excessive = swap_in > 512 * 1048576 or swap_out > 512 * 1048576 return excessive, swap_in, swap_out @property def have_resource_usage(self): """Whether resource usage is available.""" return self.resources.start_time is not None def get_resource_usage(self): """ Produce a data structure containing the low-level resource usage information. This data structure can e.g. be serialized into JSON and saved for subsequent analysis. If no resource usage is available, None is returned. """ if not self.have_resource_usage: return None cpu_percent = self.resources.aggregate_cpu_percent(phase=None, per_cpu=False) cpu_times = self.resources.aggregate_cpu_times(phase=None, per_cpu=False) io = self.resources.aggregate_io(phase=None) o = dict(version=3, argv=sys.argv, start=self.start_time, end=self.end_time, duration=self.end_time - self.start_time, resources=[], cpu_percent=cpu_percent, cpu_times=cpu_times, io=io, objects=self.build_objects) o['tiers'] = self.tiers.tiered_resource_usage() self.tiers.add_resource_fields_to_dict(o) for usage in self.resources.range_usage(): cpu_percent = self.resources.aggregate_cpu_percent(usage.start, usage.end, per_cpu=False) cpu_times = self.resources.aggregate_cpu_times(usage.start, usage.end, per_cpu=False) entry = dict( start=usage.start, end=usage.end, virt=list(usage.virt), swap=list(usage.swap), ) self.tiers.add_resources_to_dict(entry, start=usage.start, end=usage.end) o['resources'].append(entry) # If the imports for this file ran before the in-tree virtualenv # was bootstrapped (for instance, for a clobber build in automation), # psutil might not be available. # # Treat psutil as optional to avoid an outright failure to log resources # TODO: it would be nice to collect data on the storage device as well # in this case. o['system'] = {} if psutil: o['system'].update( dict( logical_cpu_count=psutil.cpu_count(), physical_cpu_count=psutil.cpu_count(logical=False), swap_total=psutil.swap_memory()[0], vmem_total=psutil.virtual_memory()[0], )) return o def log_resource_usage(self, usage): """Summarize the resource usage of this build in a log message.""" if not usage: return params = dict( duration=self.end_time - self.start_time, cpu_percent=usage['cpu_percent'], io_read_bytes=usage['io'].read_bytes, io_write_bytes=usage['io'].write_bytes, io_read_time=usage['io'].read_time, io_write_time=usage['io'].write_time, ) message = 'Overall system resources - Wall time: {duration:.0f}s; ' \ 'CPU: {cpu_percent:.0f}%; ' \ 'Read bytes: {io_read_bytes}; Write bytes: {io_write_bytes}; ' \ 'Read time: {io_read_time}; Write time: {io_write_time}' self.log(logging.WARNING, 'resource_usage', params, message) excessive, sin, sout = self.have_excessive_swapping() if excessive is not None and (sin or sout): sin /= 1048576 sout /= 1048576 self.log(logging.WARNING, 'swap_activity', { 'sin': sin, 'sout': sout }, 'Swap in/out (MB): {sin}/{sout}') def ccache_stats(self): ccache_stats = None try: ccache = which.which('ccache') output = subprocess.check_output([ccache, '-s']) ccache_stats = CCacheStats(output) except which.WhichError: pass except ValueError as e: self.log(logging.WARNING, 'ccache', {'msg': str(e)}, '{msg}') return ccache_stats
class BuildMonitor(MozbuildObject): """Monitors the output of the build.""" def init(self, warnings_path): """Create a new monitor. warnings_path is a path of a warnings database to use. """ self._warnings_path = warnings_path self.resources = SystemResourceMonitor(poll_interval=1.0) self._resources_started = False self.tiers = TierStatus(self.resources) self.warnings_database = WarningsDatabase() if os.path.exists(warnings_path): try: self.warnings_database.load_from_file(warnings_path) except ValueError: os.remove(warnings_path) self._warnings_collector = WarningsCollector( database=self.warnings_database, objdir=self.topobjdir) def start(self): """Record the start of the build.""" self.start_time = time.time() self._finder_start_cpu = self._get_finder_cpu_usage() def start_resource_recording(self): # This should be merged into start() once bug 892342 lands. self.resources.start() self._resources_started = True def on_line(self, line): """Consume a line of output from the build system. This will parse the line for state and determine whether more action is needed. Returns a BuildOutputResult instance. In this named tuple, warning will be an object describing a new parsed warning. Otherwise it will be None. state_changed indicates whether the build system changed state with this line. If the build system changed state, the caller may want to query this instance for the current state in order to update UI, etc. for_display is a boolean indicating whether the line is relevant to the user. This is typically used to filter whether the line should be presented to the user. """ if line.startswith('BUILDSTATUS'): args = line.split()[1:] action = args.pop(0) update_needed = True if action == 'TIERS': self.tiers.set_tiers(args) update_needed = False elif action == 'TIER_START': tier = args[0] subtiers = args[1:] self.tiers.begin_tier(tier, subtiers) elif action == 'TIER_FINISH': tier, = args self.tiers.finish_tier(tier) elif action == 'SUBTIER_START': tier, subtier = args[0:2] dirs = args[2:] self.tiers.begin_subtier(tier, subtier, dirs) elif action == 'SUBTIER_FINISH': tier, subtier = args self.tiers.finish_subtier(tier, subtier) elif action == 'TIERDIR_START': tier, subtier, d = args self.tiers.begin_dir(tier, subtier, d) elif action == 'TIERDIR_FINISH': tier, subtier, d = args self.tiers.finish_dir(tier, subtier, d) else: raise Exception('Unknown build status: %s' % action) return BuildOutputResult(None, update_needed, False) warning = None try: warning = self._warnings_collector.process_line(line) except: pass return BuildOutputResult(warning, False, True) def finish(self, record_usage=True): """Record the end of the build.""" self.end_time = time.time() if self._resources_started: self.resources.stop() self._finder_end_cpu = self._get_finder_cpu_usage() self.elapsed = self.end_time - self.start_time self.warnings_database.prune() self.warnings_database.save_to_file(self._warnings_path) if not record_usage: return usage = self.record_resource_usage() if not usage: return with open(self._get_state_filename('build_resources.json'), 'w') as fh: json.dump(usage, fh, indent=2) def _get_finder_cpu_usage(self): """Obtain the CPU usage of the Finder app on OS X. This is used to detect high CPU usage. """ if not sys.platform.startswith('darwin'): return None if not psutil: return None for proc in psutil.process_iter(): if proc.name != 'Finder': continue if proc.username != getpass.getuser(): continue # Try to isolate system finder as opposed to other "Finder" # processes. if not proc.exe.endswith( 'CoreServices/Finder.app/Contents/MacOS/Finder'): continue return proc.get_cpu_times() return None def have_high_finder_usage(self): """Determine whether there was high Finder CPU usage during the build. Returns True if there was high Finder CPU usage, False if there wasn't, or None if there is nothing to report. """ if not self._finder_start_cpu: return None, None # We only measure if the measured range is sufficiently long. if self.elapsed < 15: return None, None if not self._finder_end_cpu: return None, None start = self._finder_start_cpu end = self._finder_end_cpu start_total = start.user + start.system end_total = end.user + end.system cpu_seconds = end_total - start_total # If Finder used more than 25% of 1 core during the build, report an # error. finder_percent = cpu_seconds / self.elapsed * 100 return finder_percent > 25, finder_percent @property def have_resource_usage(self): """Whether resource usage is available.""" return self.resources.start_time is not None def record_resource_usage(self): """Record the resource usage of this build. We write a log message containing a high-level summary. We also produce a data structure containing the low-level resource usage information. This data structure can e.g. be serialized into JSON and saved for subsequent analysis. If no resource usage is available, None is returned. """ if not self.have_resource_usage: return None cpu_percent = self.resources.aggregate_cpu_percent(phase=None, per_cpu=False) cpu_times = self.resources.aggregate_cpu_times(phase=None, per_cpu=False) io = self.resources.aggregate_io(phase=None) self._log_resource_usage('Overall system resources', 'resource_usage', self.end_time - self.start_time, cpu_percent, cpu_times, io) o = dict( version=1, start=self.start_time, end=self.end_time, duration=self.end_time - self.start_time, resources=[], cpu_percent=cpu_percent, cpu_times=cpu_times, io=io, ) o['tiers'] = self.tiers.tiered_resource_usage() self.tiers.add_resource_fields_to_dict(o) for usage in self.resources.range_usage(): cpu_percent = self.resources.aggregate_cpu_percent(usage.start, usage.end, per_cpu=False) cpu_times = self.resources.aggregate_cpu_times(usage.start, usage.end, per_cpu=False) entry = dict( start=usage.start, end=usage.end, virt=list(usage.virt), swap=list(usage.swap), ) self.tiers.add_resources_to_dict(entry, start=usage.start, end=usage.end) o['resources'].append(entry) return o def _log_resource_usage(self, prefix, m_type, duration, cpu_percent, cpu_times, io, extra_params={}): params = dict( duration=duration, cpu_percent=cpu_percent, io_reads=io.read_count, io_writes=io.write_count, io_read_bytes=io.read_bytes, io_write_bytes=io.write_bytes, io_read_time=io.read_time, io_write_time=io.write_time, ) params.update(extra_params) message = prefix + ' - Wall time: {duration:.0f}s; ' \ 'CPU: {cpu_percent:.0f}%; ' \ 'Read bytes: {io_read_bytes}; Write bytes: {io_write_bytes}; ' \ 'Read time: {io_read_time}; Write time: {io_write_time}' self.log(logging.WARNING, m_type, params, message)
class BuildMonitor(MozbuildObject): """Monitors the output of the build.""" def init(self, warnings_path): """Create a new monitor. warnings_path is a path of a warnings database to use. """ self._warnings_path = warnings_path self.resources = SystemResourceMonitor(poll_interval=1.0) self._resources_started = False self.tiers = TierStatus(self.resources) self.warnings_database = WarningsDatabase() if os.path.exists(warnings_path): try: self.warnings_database.load_from_file(warnings_path) except ValueError: os.remove(warnings_path) self._warnings_collector = WarningsCollector(database=self.warnings_database, objdir=self.topobjdir) self.build_objects = [] def start(self): """Record the start of the build.""" self.start_time = time.time() self._finder_start_cpu = self._get_finder_cpu_usage() def start_resource_recording(self): # This should be merged into start() once bug 892342 lands. self.resources.start() self._resources_started = True def on_line(self, line): """Consume a line of output from the build system. This will parse the line for state and determine whether more action is needed. Returns a BuildOutputResult instance. In this named tuple, warning will be an object describing a new parsed warning. Otherwise it will be None. state_changed indicates whether the build system changed state with this line. If the build system changed state, the caller may want to query this instance for the current state in order to update UI, etc. for_display is a boolean indicating whether the line is relevant to the user. This is typically used to filter whether the line should be presented to the user. """ if line.startswith("BUILDSTATUS"): args = line.split()[1:] action = args.pop(0) update_needed = True if action == "TIERS": self.tiers.set_tiers(args) update_needed = False elif action == "TIER_START": tier = args[0] self.tiers.begin_tier(tier) elif action == "TIER_FINISH": tier, = args self.tiers.finish_tier(tier) elif action == "OBJECT_FILE": self.build_objects.append(args[0]) update_needed = False else: raise Exception("Unknown build status: %s" % action) return BuildOutputResult(None, update_needed, False) warning = None try: warning = self._warnings_collector.process_line(line) except: pass return BuildOutputResult(warning, False, True) def stop_resource_recording(self): if self._resources_started: self.resources.stop() self._resources_started = False def finish(self, record_usage=True): """Record the end of the build.""" self.stop_resource_recording() self.end_time = time.time() self._finder_end_cpu = self._get_finder_cpu_usage() self.elapsed = self.end_time - self.start_time self.warnings_database.prune() self.warnings_database.save_to_file(self._warnings_path) if not record_usage: return try: usage = self.get_resource_usage() if not usage: return self.log_resource_usage(usage) with open(self._get_state_filename("build_resources.json"), "w") as fh: json.dump(self.resources.as_dict(), fh, indent=2) except Exception as e: self.log( logging.WARNING, "build_resources_error", {"msg": str(e)}, "Exception when writing resource usage file: {msg}", ) def _get_finder_cpu_usage(self): """Obtain the CPU usage of the Finder app on OS X. This is used to detect high CPU usage. """ if not sys.platform.startswith("darwin"): return None if not psutil: return None for proc in psutil.process_iter(): if proc.name != "Finder": continue if proc.username != getpass.getuser(): continue # Try to isolate system finder as opposed to other "Finder" # processes. if not proc.exe.endswith("CoreServices/Finder.app/Contents/MacOS/Finder"): continue return proc.get_cpu_times() return None def have_high_finder_usage(self): """Determine whether there was high Finder CPU usage during the build. Returns True if there was high Finder CPU usage, False if there wasn't, or None if there is nothing to report. """ if not self._finder_start_cpu: return None, None # We only measure if the measured range is sufficiently long. if self.elapsed < 15: return None, None if not self._finder_end_cpu: return None, None start = self._finder_start_cpu end = self._finder_end_cpu start_total = start.user + start.system end_total = end.user + end.system cpu_seconds = end_total - start_total # If Finder used more than 25% of 1 core during the build, report an # error. finder_percent = cpu_seconds / self.elapsed * 100 return finder_percent > 25, finder_percent def have_excessive_swapping(self): """Determine whether there was excessive swapping during the build. Returns a tuple of (excessive, swap_in, swap_out). All values are None if no swap information is available. """ if not self.have_resource_usage: return None, None, None swap_in = sum(m.swap.sin for m in self.resources.measurements) swap_out = sum(m.swap.sout for m in self.resources.measurements) # The threshold of 1024 MB has been arbitrarily chosen. # # Choosing a proper value that is ideal for everyone is hard. We will # likely iterate on the logic until people are generally satisfied. # If a value is too low, the eventual warning produced does not carry # much meaning. If the threshold is too high, people may not see the # warning and the warning will thus be ineffective. excessive = swap_in > 512 * 1048576 or swap_out > 512 * 1048576 return excessive, swap_in, swap_out @property def have_resource_usage(self): """Whether resource usage is available.""" return self.resources.start_time is not None def get_resource_usage(self): """ Produce a data structure containing the low-level resource usage information. This data structure can e.g. be serialized into JSON and saved for subsequent analysis. If no resource usage is available, None is returned. """ if not self.have_resource_usage: return None cpu_percent = self.resources.aggregate_cpu_percent(phase=None, per_cpu=False) cpu_times = self.resources.aggregate_cpu_times(phase=None, per_cpu=False) io = self.resources.aggregate_io(phase=None) o = dict( version=3, argv=sys.argv, start=self.start_time, end=self.end_time, duration=self.end_time - self.start_time, resources=[], cpu_percent=cpu_percent, cpu_times=cpu_times, io=io, objects=self.build_objects, ) o["tiers"] = self.tiers.tiered_resource_usage() self.tiers.add_resource_fields_to_dict(o) for usage in self.resources.range_usage(): cpu_percent = self.resources.aggregate_cpu_percent(usage.start, usage.end, per_cpu=False) cpu_times = self.resources.aggregate_cpu_times(usage.start, usage.end, per_cpu=False) entry = dict(start=usage.start, end=usage.end, virt=list(usage.virt), swap=list(usage.swap)) self.tiers.add_resources_to_dict(entry, start=usage.start, end=usage.end) o["resources"].append(entry) # If the imports for this file ran before the in-tree virtualenv # was bootstrapped (for instance, for a clobber build in automation), # psutil might not be available. # # Treat psutil as optional to avoid an outright failure to log resources # TODO: it would be nice to collect data on the storage device as well # in this case. o["system"] = {} if psutil: o["system"].update( dict( logical_cpu_count=psutil.cpu_count(), physical_cpu_count=psutil.cpu_count(logical=False), swap_total=psutil.swap_memory()[0], vmem_total=psutil.virtual_memory()[0], ) ) return o def log_resource_usage(self, usage): """Summarize the resource usage of this build in a log message.""" if not usage: return params = dict( duration=self.end_time - self.start_time, cpu_percent=usage["cpu_percent"], io_read_bytes=usage["io"].read_bytes, io_write_bytes=usage["io"].write_bytes, io_read_time=usage["io"].read_time, io_write_time=usage["io"].write_time, ) message = ( "Overall system resources - Wall time: {duration:.0f}s; " "CPU: {cpu_percent:.0f}%; " "Read bytes: {io_read_bytes}; Write bytes: {io_write_bytes}; " "Read time: {io_read_time}; Write time: {io_write_time}" ) self.log(logging.WARNING, "resource_usage", params, message) excessive, sin, sout = self.have_excessive_swapping() if excessive is not None and (sin or sout): sin /= 1048576 sout /= 1048576 self.log(logging.WARNING, "swap_activity", {"sin": sin, "sout": sout}, "Swap in/out (MB): {sin}/{sout}") def ccache_stats(self): ccache_stats = None try: ccache = which.which("ccache") output = subprocess.check_output([ccache, "-s"]) ccache_stats = CCacheStats(output) except which.WhichError: pass except ValueError as e: self.log(logging.WARNING, "ccache", {"msg": str(e)}, "{msg}") return ccache_stats