def test_ramp_boost(self,
nrg_threshold_pct=0.1,
bad_samples_threshold_pct=0.1) -> ResultBundle:
"""
Test that the energy boost feature is triggering as expected.
"""
# If there was no cost_margin sample to look at, that means boosting
# was not exhibited by that test so we cannot conclude anything
df = self.df_ramp_boost()
self._plot_test_boost(df)
if df.empty:
return ResultBundle(Result.UNDECIDED)
# Make sure the boost is always positive (negative cannot really happen
# since the kernel is using unsigned arithmetic, but still check in
# case there are some dataframe handling issues)
assert not (df['expected_cost_margin'] < 0).any()
assert not (df['cost_margin'] < 0).any()
# "rect" method is accurate here since the signal is really following
# "post" steps
expected_boost_nrg = series_mean(df['expected_cost_margin'])
actual_boost_nrg = series_mean(df['cost_margin'])
# Check that the total amount of boost is close to expectations
lower = max(0, expected_boost_nrg - nrg_threshold_pct)
higher = expected_boost_nrg
passed_overhead = lower <= actual_boost_nrg <= higher
# Check the shape of the signal: actual boost must be lower or equal
# than the expected one.
good_shape_nr = (df['cost_margin'] <= df['expected_cost_margin']).sum()
df_len = len(df)
bad_shape_nr = df_len - good_shape_nr
bad_shape_pct = bad_shape_nr / df_len * 100
# Tolerate a few bad samples that added too much boost
passed_shape = bad_shape_pct < bad_samples_threshold_pct
passed = passed_overhead and passed_shape
res = ResultBundle.from_bool(passed)
res.add_metric('expected boost energy overhead', expected_boost_nrg,
'%')
res.add_metric('boost energy overhead', actual_boost_nrg, '%')
res.add_metric('bad boost samples', bad_shape_pct, '%')
# Add some slack metrics and plots
analysis = self.trace.analysis.rta
for task in self.rtapp_tasks:
analysis.plot_slack_histogram(task)
analysis.plot_perf_index_histogram(task)
analysis.plot_latency(task)
res.add_metric('avg slack', self.get_avg_slack(), 'us')
res.add_metric('avg negative slack',
self.get_avg_slack(only_negative=True), 'us')
return res
def _test_correctness(self, signal_name, mean_error_margin_pct,
max_error_margin_pct):
task = self.task_name
df = self.get_simulated_pelt(task, signal_name)
abs_error = df['error'].abs()
mean_error_pct = series_mean(abs_error) / UTIL_SCALE * 100
max_error_pct = abs_error.max() / UTIL_SCALE * 100
mean_ok = mean_error_pct <= mean_error_margin_pct
max_ok = max_error_pct <= max_error_margin_pct
res = ResultBundle.from_bool(mean_ok and max_ok)
res.add_metric('actual mean', series_mean(df[signal_name]))
res.add_metric('simulated mean', series_mean(df['simulated']))
res.add_metric('mean error', mean_error_pct, '%')
res.add_metric('actual max', df[signal_name].max())
res.add_metric('simulated max', df['simulated'].max())
res.add_metric('max error', max_error_pct, '%')
self._plot_pelt(task, signal_name, df['simulated'], 'correctness')
res = self._add_cpu_metric(res)
return res
def get_trace_cpu_util(self):
"""
Get the per-phase average CPU utilization read from the trace
:returns: A dict of the shape {cpu : {phase_id : trace_util}}
"""
df = self.trace.analysis.load_tracking.df_cpus_signal('util')
phase_start = self.trace.start
cpu_util = {}
for i, phase in enumerate(self.reference_task.phases):
# Start looking at signals once they should've converged
start = phase_start + UTIL_AVG_CONVERGENCE_TIME_S
# Trim the end a bit, otherwise we could have one or two events
# from the next phase
end = phase_start + phase.duration_s * .9
phase_df = df[start:end]
for cpu in self.cpus:
util = phase_df[phase_df.cpu == cpu].util
# The runqueue util signal's average does not match the duty
# cycle of the task, since it "decays instantly" at next task
# wakeup, but stays at its previous value when the task sleeps.
# This means that rq PELT signal average is higher than the
# idealized PELT signal. Using trapz integration allows to
# lower the contribution of the sleep-time util, since it links
# with a straight line the point when task goes to sleep with
# the wakeup util point.
cpu_util.setdefault(cpu, {})[i] = series_mean(util,
method='trapz')
phase_start += phase.duration_s
return cpu_util
def _test_task_signal(self, signal_name, allowed_error_pct, trace, cpu,
task_name, capacity):
# Use utilization signal for both load and util, since they should be
# proportionnal in the test environment we setup
exp_signal = self.get_expected_util_avg(trace, cpu, task_name,
capacity)
signal_df = self.get_task_sched_signal(trace, cpu, task_name,
signal_name)
signal = signal_df[UTIL_AVG_CONVERGENCE_TIME_S:][signal_name]
signal_mean = series_mean(signal)
# Since load is now CPU invariant in recent kernel versions, we don't
# rescale it back. To match the old behavior, that line is
# needed:
# exp_signal /= (self.plat_info['cpu-capacities'][cpu] / UTIL_SCALE)
kernel_version = self.plat_info['kernel']['version']
if (signal_name == 'load' and kernel_version.parts[:2] < (5, 1)):
self.get_logger().warning(
'Load signal is assumed to be CPU invariant, which is true for recent mainline kernels, but may be wrong for {}'
.format(kernel_version, ))
ok = self.is_almost_equal(exp_signal, signal_mean, allowed_error_pct)
return ok, exp_signal, signal_mean
def get_average_cpu_frequency(self, cpu):
"""
Get the average frequency for a given CPU
:param cpu: The CPU to analyse
:type cpu: int
"""
df = self.df_cpu_frequency(cpu)
freq = series_refit_index(df['frequency'], window=self.trace.window)
return series_mean(freq)
def compute_means(row):
start = row.name
end = start + row['duration']
phase_activations = df_window(df_activations, (start, end))
phase_util = df_window(df_util, (start, end))
series = pd.Series({
'Phase duty cycle average': series_mean(phase_activations['duty_cycle']),
'Phase util tunnel average': kernel_util_mean(
phase_util['util'],
plat_info=self.plat_info,
),
})
return series
def plot_peripheral_frequency(self, clk_name: str, average: bool = True):
"""
Plot frequency for the specified peripheral clock frequency
:param clk_name: The clock name for which to plot frequency
:type clk_name: str
:param average: If ``True``, add a horizontal line which is the
frequency average.
:type average: bool
"""
df = self.df_peripheral_clock_effective_rate(clk_name)
freq = df['effective_rate']
freq = series_refit_index(freq, window=self.trace.window)
fig = plot_signal(freq, name=f'Frequency of {clk_name} (Hz)')
if average:
avg = series_mean(freq)
if avg > 0:
fig *= hv.HLine(avg, group='average').opts(color='red')
return fig
def test_means(self) -> ResultBundle:
"""
Test signals are properly "dominated".
The mean of `enqueued` is expected to be always not
smaller than that of `util`, since this last is subject to decays
while the first not.
The mean of `enqueued` is expected to be always greater or
equal than the mean of `util`, since this `util` is subject
to decays while `enqueued` not.
On fast-ramp systems, the `ewma` signal is never smaller then
the `enqueued`, thus his mean is expected to be bigger.
On non fast-ramp systems instead, the `ewma` is expected to be
smaller then `enqueued` in ramp-up phases, or bigger in
ramp-down phases.
Those conditions are checked on a single execution of a task which has
three main behaviours:
* STABLE: periodic big task running for a relatively long period to
ensure `util` saturation.
* DOWN: periodic ramp-down task, to slowly decay `util`
* UP: periodic ramp-up task, to slowly increase `util`
"""
failure_reasons = {}
metrics = {}
task = self.rtapp_task_ids_map['test'][0]
ue_df = self.trace.df_event('sched_util_est_se')
ue_df = df_filter_task_ids(ue_df, [task])
ua_df = self.trace.ana.load_tracking.df_task_signal(task, 'util')
failures = []
for phase in self.trace.ana.rta.task_phase_windows(
task, wlgen_profile=self.rtapp_profile):
if not phase.properties['meta']['from_test']:
continue
apply_phase_window = functools.partial(df_refit_index,
window=(phase.start,
phase.end))
ue_phase_df = apply_phase_window(ue_df)
mean_enqueued = series_mean(ue_phase_df['enqueued'])
mean_ewma = series_mean(ue_phase_df['ewma'])
ua_phase_df = apply_phase_window(ua_df)
mean_util = series_mean(ua_phase_df['util'])
def make_issue(msg):
return msg.format(
util=f'util={mean_util}',
enq=f'enqueued={mean_enqueued}',
ewma=f'ewma={mean_ewma}',
)
issue = None
if mean_enqueued < mean_util:
issue = make_issue('{enq} smaller than {util}')
# Running on FastRamp kernels:
elif self.fast_ramp:
# STABLE, DOWN and UP:
if mean_ewma < mean_enqueued:
issue = make_issue(
'no fast ramp: {ewma} smaller than {enq}')
# Running on (legacy) non FastRamp kernels:
else:
# STABLE: ewma ramping up
if phase.id.startswith('test/stable'):
if mean_ewma > mean_enqueued:
issue = make_issue(
'fast ramp, stable: {ewma} bigger than {enq}')
# DOWN: ewma ramping down
elif phase.id.startswith('test/ramp_down'):
if mean_ewma < mean_enqueued:
issue = make_issue(
'fast ramp, down: {ewma} smaller than {enq}')
# UP: ewma ramping up
elif phase.id.startswith('test/ramp_up'):
if mean_ewma > mean_enqueued:
issue = make_issue(
'fast ramp, up: {ewma} bigger than {enq}')
metrics[phase.id] = PhaseStats(phase.start, phase.end, mean_util,
mean_enqueued, mean_ewma, issue)
failures = [(phase, stat) for phase, stat in metrics.items()
if stat.issue]
# Plot signals to support debugging analysis
self._plot_signals(task, 'means',
sorted(stat.start for phase, stat in failures))
bundle = ResultBundle.from_bool(not failures)
bundle.add_metric("fast ramp", self.fast_ramp)
bundle.add_metric("phases", metrics)
bundle.add_metric("failures",
sorted(phase for phase, stat in failures))
return bundle
def get_expected_cpu_util(self):
"""
Get the per-phase average CPU utilization expected from the duty cycle
of the tasks found in the trace.
:returns: A dict of the shape {cpu : {phase_id : expected_util}}
.. note:: This is more robust than just looking at the duty cycle in
the task profile, since rtapp might not reproduce accurately the
duty cycle it was asked.
"""
cpu_capacities = self.plat_info['cpu-capacities']['rtapp']
cpu_util = {}
cpu_freqs = self.plat_info['freqs']
try:
freq_df = self.trace.analysis.frequency.df_cpus_frequency()
except MissingTraceEventError:
cpus_rel_freq = None
else:
cpus_rel_freq = {
# Frequency, normalized according to max frequency on that CPU
cols['cpu']: df['frequency'] / max(cpu_freqs[cols['cpu']])
for cols, df in df_split_signals(freq_df, ['cpu'])
}
for task in self.rtapp_task_ids:
df = self.trace.analysis.tasks.df_task_activation(task)
for row in self.trace.analysis.rta.df_phases(task).itertuples():
phase = row.phase
duration = row.duration
start = row.Index
end = start + duration
# Ignore the first quarter of the util signal of each phase, since
# it's impacted by the phase change, and util can be affected
# (rtapp does some bookkeeping at the beginning of phases)
# start += duration / 4
# readjust the duration to take into account the modification of start
duration = end - start
window = (start, end)
phase_df = df_window(df, window, clip_window=True)
for cpu in self.cpus:
if cpus_rel_freq is None:
rel_freq_mean = 1
else:
phase_freq_series = df_window(cpus_rel_freq[cpu],
window=window,
clip_window=True)
# # We might not have frequency data at the beginning of the
# # trace, or if not frequency transition happened at all.
if phase_freq_series.empty:
rel_freq_mean = 1
else:
# If we lack freq data at the beginning of the
# window, assume the frequency was right.
if phase_freq_series.index[0] > start:
phase_freq_series = pd.concat([
pd.Series([1.0], index=[start]),
phase_freq_series
])
# Extend the frequency to the right so that the mean
# takes into account all the data we have
freq_window = (phase_freq_series.index[0], end)
rel_freq_mean = series_mean(
series_refit_index(phase_freq_series,
window=freq_window))
cpu_phase_df = phase_df[phase_df['cpu'] == cpu].dropna()
if cpu_phase_df.empty:
duty_cycle = 0
cpu_residency = 0
else:
duty_cycle = series_mean(
df_refit_index(cpu_phase_df['duty_cycle'],
window=window))
cpu_residency = end - max(cpu_phase_df.index[0], start)
phase_util = UTIL_SCALE * duty_cycle * (
cpu_capacities[cpu] / UTIL_SCALE)
# Pro-rata with the time spent on that CPU, so we get
# the correct average.
phase_util *= cpu_residency / duration
# We might not have run at max freq, e.g. because of
# thermal capping, so take that into account
phase_util *= rel_freq_mean
cpu_util.setdefault(cpu, {}).setdefault(phase, 0)
cpu_util[cpu][phase] += phase_util
return cpu_util
