def test_ramp_boost(self, nrg_threshold_pct=0.1, bad_samples_threshold_pct=0.1) -> ResultBundle: """ Test that the energy boost feature is triggering as expected. """ # If there was no cost_margin sample to look at, that means boosting # was not exhibited by that test so we cannot conclude anything df = self.df_ramp_boost() self._plot_test_boost(df) if df.empty: return ResultBundle(Result.UNDECIDED) # Make sure the boost is always positive (negative cannot really happen # since the kernel is using unsigned arithmetic, but still check in # case there are some dataframe handling issues) assert not (df['expected_cost_margin'] < 0).any() assert not (df['cost_margin'] < 0).any() # "rect" method is accurate here since the signal is really following # "post" steps expected_boost_nrg = series_mean(df['expected_cost_margin']) actual_boost_nrg = series_mean(df['cost_margin']) # Check that the total amount of boost is close to expectations lower = max(0, expected_boost_nrg - nrg_threshold_pct) higher = expected_boost_nrg passed_overhead = lower <= actual_boost_nrg <= higher # Check the shape of the signal: actual boost must be lower or equal # than the expected one. good_shape_nr = (df['cost_margin'] <= df['expected_cost_margin']).sum() df_len = len(df) bad_shape_nr = df_len - good_shape_nr bad_shape_pct = bad_shape_nr / df_len * 100 # Tolerate a few bad samples that added too much boost passed_shape = bad_shape_pct < bad_samples_threshold_pct passed = passed_overhead and passed_shape res = ResultBundle.from_bool(passed) res.add_metric('expected boost energy overhead', expected_boost_nrg, '%') res.add_metric('boost energy overhead', actual_boost_nrg, '%') res.add_metric('bad boost samples', bad_shape_pct, '%') # Add some slack metrics and plots analysis = self.trace.analysis.rta for task in self.rtapp_tasks: analysis.plot_slack_histogram(task) analysis.plot_perf_index_histogram(task) analysis.plot_latency(task) res.add_metric('avg slack', self.get_avg_slack(), 'us') res.add_metric('avg negative slack', self.get_avg_slack(only_negative=True), 'us') return res
def _test_correctness(self, signal_name, mean_error_margin_pct, max_error_margin_pct): task = self.task_name df = self.get_simulated_pelt(task, signal_name) abs_error = df['error'].abs() mean_error_pct = series_mean(abs_error) / UTIL_SCALE * 100 max_error_pct = abs_error.max() / UTIL_SCALE * 100 mean_ok = mean_error_pct <= mean_error_margin_pct max_ok = max_error_pct <= max_error_margin_pct res = ResultBundle.from_bool(mean_ok and max_ok) res.add_metric('actual mean', series_mean(df[signal_name])) res.add_metric('simulated mean', series_mean(df['simulated'])) res.add_metric('mean error', mean_error_pct, '%') res.add_metric('actual max', df[signal_name].max()) res.add_metric('simulated max', df['simulated'].max()) res.add_metric('max error', max_error_pct, '%') self._plot_pelt(task, signal_name, df['simulated'], 'correctness') res = self._add_cpu_metric(res) return res
def get_trace_cpu_util(self): """ Get the per-phase average CPU utilization read from the trace :returns: A dict of the shape {cpu : {phase_id : trace_util}} """ df = self.trace.analysis.load_tracking.df_cpus_signal('util') phase_start = self.trace.start cpu_util = {} for i, phase in enumerate(self.reference_task.phases): # Start looking at signals once they should've converged start = phase_start + UTIL_AVG_CONVERGENCE_TIME_S # Trim the end a bit, otherwise we could have one or two events # from the next phase end = phase_start + phase.duration_s * .9 phase_df = df[start:end] for cpu in self.cpus: util = phase_df[phase_df.cpu == cpu].util # The runqueue util signal's average does not match the duty # cycle of the task, since it "decays instantly" at next task # wakeup, but stays at its previous value when the task sleeps. # This means that rq PELT signal average is higher than the # idealized PELT signal. Using trapz integration allows to # lower the contribution of the sleep-time util, since it links # with a straight line the point when task goes to sleep with # the wakeup util point. cpu_util.setdefault(cpu, {})[i] = series_mean(util, method='trapz') phase_start += phase.duration_s return cpu_util
def _test_task_signal(self, signal_name, allowed_error_pct, trace, cpu, task_name, capacity): # Use utilization signal for both load and util, since they should be # proportionnal in the test environment we setup exp_signal = self.get_expected_util_avg(trace, cpu, task_name, capacity) signal_df = self.get_task_sched_signal(trace, cpu, task_name, signal_name) signal = signal_df[UTIL_AVG_CONVERGENCE_TIME_S:][signal_name] signal_mean = series_mean(signal) # Since load is now CPU invariant in recent kernel versions, we don't # rescale it back. To match the old behavior, that line is # needed: # exp_signal /= (self.plat_info['cpu-capacities'][cpu] / UTIL_SCALE) kernel_version = self.plat_info['kernel']['version'] if (signal_name == 'load' and kernel_version.parts[:2] < (5, 1)): self.get_logger().warning( 'Load signal is assumed to be CPU invariant, which is true for recent mainline kernels, but may be wrong for {}' .format(kernel_version, )) ok = self.is_almost_equal(exp_signal, signal_mean, allowed_error_pct) return ok, exp_signal, signal_mean
def get_average_cpu_frequency(self, cpu): """ Get the average frequency for a given CPU :param cpu: The CPU to analyse :type cpu: int """ df = self.df_cpu_frequency(cpu) freq = series_refit_index(df['frequency'], window=self.trace.window) return series_mean(freq)
def compute_means(row): start = row.name end = start + row['duration'] phase_activations = df_window(df_activations, (start, end)) phase_util = df_window(df_util, (start, end)) series = pd.Series({ 'Phase duty cycle average': series_mean(phase_activations['duty_cycle']), 'Phase util tunnel average': kernel_util_mean( phase_util['util'], plat_info=self.plat_info, ), }) return series
def plot_peripheral_frequency(self, clk_name: str, average: bool = True): """ Plot frequency for the specified peripheral clock frequency :param clk_name: The clock name for which to plot frequency :type clk_name: str :param average: If ``True``, add a horizontal line which is the frequency average. :type average: bool """ df = self.df_peripheral_clock_effective_rate(clk_name) freq = df['effective_rate'] freq = series_refit_index(freq, window=self.trace.window) fig = plot_signal(freq, name=f'Frequency of {clk_name} (Hz)') if average: avg = series_mean(freq) if avg > 0: fig *= hv.HLine(avg, group='average').opts(color='red') return fig
def test_means(self) -> ResultBundle: """ Test signals are properly "dominated". The mean of `enqueued` is expected to be always not smaller than that of `util`, since this last is subject to decays while the first not. The mean of `enqueued` is expected to be always greater or equal than the mean of `util`, since this `util` is subject to decays while `enqueued` not. On fast-ramp systems, the `ewma` signal is never smaller then the `enqueued`, thus his mean is expected to be bigger. On non fast-ramp systems instead, the `ewma` is expected to be smaller then `enqueued` in ramp-up phases, or bigger in ramp-down phases. Those conditions are checked on a single execution of a task which has three main behaviours: * STABLE: periodic big task running for a relatively long period to ensure `util` saturation. * DOWN: periodic ramp-down task, to slowly decay `util` * UP: periodic ramp-up task, to slowly increase `util` """ failure_reasons = {} metrics = {} task = self.rtapp_task_ids_map['test'][0] ue_df = self.trace.df_event('sched_util_est_se') ue_df = df_filter_task_ids(ue_df, [task]) ua_df = self.trace.ana.load_tracking.df_task_signal(task, 'util') failures = [] for phase in self.trace.ana.rta.task_phase_windows( task, wlgen_profile=self.rtapp_profile): if not phase.properties['meta']['from_test']: continue apply_phase_window = functools.partial(df_refit_index, window=(phase.start, phase.end)) ue_phase_df = apply_phase_window(ue_df) mean_enqueued = series_mean(ue_phase_df['enqueued']) mean_ewma = series_mean(ue_phase_df['ewma']) ua_phase_df = apply_phase_window(ua_df) mean_util = series_mean(ua_phase_df['util']) def make_issue(msg): return msg.format( util=f'util={mean_util}', enq=f'enqueued={mean_enqueued}', ewma=f'ewma={mean_ewma}', ) issue = None if mean_enqueued < mean_util: issue = make_issue('{enq} smaller than {util}') # Running on FastRamp kernels: elif self.fast_ramp: # STABLE, DOWN and UP: if mean_ewma < mean_enqueued: issue = make_issue( 'no fast ramp: {ewma} smaller than {enq}') # Running on (legacy) non FastRamp kernels: else: # STABLE: ewma ramping up if phase.id.startswith('test/stable'): if mean_ewma > mean_enqueued: issue = make_issue( 'fast ramp, stable: {ewma} bigger than {enq}') # DOWN: ewma ramping down elif phase.id.startswith('test/ramp_down'): if mean_ewma < mean_enqueued: issue = make_issue( 'fast ramp, down: {ewma} smaller than {enq}') # UP: ewma ramping up elif phase.id.startswith('test/ramp_up'): if mean_ewma > mean_enqueued: issue = make_issue( 'fast ramp, up: {ewma} bigger than {enq}') metrics[phase.id] = PhaseStats(phase.start, phase.end, mean_util, mean_enqueued, mean_ewma, issue) failures = [(phase, stat) for phase, stat in metrics.items() if stat.issue] # Plot signals to support debugging analysis self._plot_signals(task, 'means', sorted(stat.start for phase, stat in failures)) bundle = ResultBundle.from_bool(not failures) bundle.add_metric("fast ramp", self.fast_ramp) bundle.add_metric("phases", metrics) bundle.add_metric("failures", sorted(phase for phase, stat in failures)) return bundle
def get_expected_cpu_util(self): """ Get the per-phase average CPU utilization expected from the duty cycle of the tasks found in the trace. :returns: A dict of the shape {cpu : {phase_id : expected_util}} .. note:: This is more robust than just looking at the duty cycle in the task profile, since rtapp might not reproduce accurately the duty cycle it was asked. """ cpu_capacities = self.plat_info['cpu-capacities']['rtapp'] cpu_util = {} cpu_freqs = self.plat_info['freqs'] try: freq_df = self.trace.analysis.frequency.df_cpus_frequency() except MissingTraceEventError: cpus_rel_freq = None else: cpus_rel_freq = { # Frequency, normalized according to max frequency on that CPU cols['cpu']: df['frequency'] / max(cpu_freqs[cols['cpu']]) for cols, df in df_split_signals(freq_df, ['cpu']) } for task in self.rtapp_task_ids: df = self.trace.analysis.tasks.df_task_activation(task) for row in self.trace.analysis.rta.df_phases(task).itertuples(): phase = row.phase duration = row.duration start = row.Index end = start + duration # Ignore the first quarter of the util signal of each phase, since # it's impacted by the phase change, and util can be affected # (rtapp does some bookkeeping at the beginning of phases) # start += duration / 4 # readjust the duration to take into account the modification of start duration = end - start window = (start, end) phase_df = df_window(df, window, clip_window=True) for cpu in self.cpus: if cpus_rel_freq is None: rel_freq_mean = 1 else: phase_freq_series = df_window(cpus_rel_freq[cpu], window=window, clip_window=True) # # We might not have frequency data at the beginning of the # # trace, or if not frequency transition happened at all. if phase_freq_series.empty: rel_freq_mean = 1 else: # If we lack freq data at the beginning of the # window, assume the frequency was right. if phase_freq_series.index[0] > start: phase_freq_series = pd.concat([ pd.Series([1.0], index=[start]), phase_freq_series ]) # Extend the frequency to the right so that the mean # takes into account all the data we have freq_window = (phase_freq_series.index[0], end) rel_freq_mean = series_mean( series_refit_index(phase_freq_series, window=freq_window)) cpu_phase_df = phase_df[phase_df['cpu'] == cpu].dropna() if cpu_phase_df.empty: duty_cycle = 0 cpu_residency = 0 else: duty_cycle = series_mean( df_refit_index(cpu_phase_df['duty_cycle'], window=window)) cpu_residency = end - max(cpu_phase_df.index[0], start) phase_util = UTIL_SCALE * duty_cycle * ( cpu_capacities[cpu] / UTIL_SCALE) # Pro-rata with the time spent on that CPU, so we get # the correct average. phase_util *= cpu_residency / duration # We might not have run at max freq, e.g. because of # thermal capping, so take that into account phase_util *= rel_freq_mean cpu_util.setdefault(cpu, {}).setdefault(phase, 0) cpu_util[cpu][phase] += phase_util return cpu_util