def test_ramp_boost(self,
nrg_threshold_pct=0.1,
bad_samples_threshold_pct=0.1) -> ResultBundle:
"""
Test that the energy boost feature is triggering as expected.
"""
# If there was no cost_margin sample to look at, that means boosting
# was not exhibited by that test so we cannot conclude anything
df = self.df_ramp_boost()
self._plot_test_boost(df)
if df.empty:
return ResultBundle(Result.UNDECIDED)
# Make sure the boost is always positive (negative cannot really happen
# since the kernel is using unsigned arithmetic, but still check in
# case there are some dataframe handling issues)
assert not (df['expected_cost_margin'] < 0).any()
assert not (df['cost_margin'] < 0).any()
# "rect" method is accurate here since the signal is really following
# "post" steps
expected_boost_nrg = series_mean(df['expected_cost_margin'])
actual_boost_nrg = series_mean(df['cost_margin'])
# Check that the total amount of boost is close to expectations
lower = max(0, expected_boost_nrg - nrg_threshold_pct)
higher = expected_boost_nrg
passed_overhead = lower <= actual_boost_nrg <= higher
# Check the shape of the signal: actual boost must be lower or equal
# than the expected one.
good_shape_nr = (df['cost_margin'] <= df['expected_cost_margin']).sum()
df_len = len(df)
bad_shape_nr = df_len - good_shape_nr
bad_shape_pct = bad_shape_nr / df_len * 100
# Tolerate a few bad samples that added too much boost
passed_shape = bad_shape_pct < bad_samples_threshold_pct
passed = passed_overhead and passed_shape
res = ResultBundle.from_bool(passed)
res.add_metric('expected boost energy overhead', expected_boost_nrg,
'%')
res.add_metric('boost energy overhead', actual_boost_nrg, '%')
res.add_metric('bad boost samples', bad_shape_pct, '%')
# Add some slack metrics and plots
analysis = self.trace.analysis.rta
for task in self.rtapp_tasks:
analysis.plot_slack_histogram(task)
analysis.plot_perf_index_histogram(task)
analysis.plot_latency(task)
res.add_metric('avg slack', self.get_avg_slack(), 'us')
res.add_metric('avg negative slack',
self.get_avg_slack(only_negative=True), 'us')
return res
def test_performance_sanity(self) -> ResultBundle:
"""
Assert that higher CPU frequency leads to more work done
"""
res = ResultBundle.from_bool(True)
cpu_items = {
cpu: {
# We expect only one item per frequency
item.freq: item
for item in freq_items
}
for cpu, freq_items in groupby(self.sanity_items, key=lambda item: item.cpu)
}
failed = []
passed = True
for cpu, freq_items in cpu_items.items():
sorted_items = sorted(freq_items.values(), key=lambda item: item.freq)
work = [item.work for item in sorted_items]
if work != sorted(work):
passed = False
failed.append(cpu)
res = ResultBundle.from_bool(passed)
work_metric = {
cpu: {freq: item.work for freq, item in freq_items.items()}
for cpu, freq_items in cpu_items.items()
}
res.add_metric('CPUs work', work_metric)
res.add_metric('Failed CPUs', failed)
return res
def check_from_target(cls, target):
super().check_from_target(target)
try:
cls._reset_fail(target, 0)
except TargetStableError:
ResultBundle.raise_skip(
"Target can't reset the hotplug fail interface")
def check_from_target(cls, target):
super().check_from_target(target)
kconfig = target.plat_info['kernel']['config']
if not kconfig.get('UCLAMP_TASK'):
ResultBundle.raise_skip(
"The target's kernel needs CONFIG_UCLAMP_TASK=y kconfig enabled"
)
def _test_signal(self, signal_name, allowed_error_pct):
passed = True
expected_data = {}
trace_data = {}
capacity = self.plat_info['cpu-capacities'][self.cpu]
# Scale the capacity linearly according to the frequency
max_freq = max(self.plat_info['freqs'][self.cpu])
capacity *= (self.freq / max_freq)
for name, task in self.rtapp_profile.items():
ok, exp_util, signal_mean = self._test_task_signal(
signal_name, allowed_error_pct, self.trace, self.cpu, name,
capacity)
if not ok:
passed = False
expected_data[name] = TestMetric(exp_util)
trace_data[name] = TestMetric(signal_mean)
freq_str = '@{}'.format(self.freq) if self.freq is not None else ''
bundle = ResultBundle.from_bool(passed)
bundle.add_metric("cpu", '{}{}'.format(self.cpu, freq_str))
bundle.add_metric("Expected signals", expected_data)
bundle.add_metric("Trace signals", trace_data)
return bundle
def test_task_placement(self, energy_est_threshold_pct=5,
nrg_model: EnergyModel = None, capacity_margin_pct=20) -> ResultBundle:
"""
Test that task placement was energy-efficient
:param nrg_model: Allow using an alternate EnergyModel instead of
``nrg_model```
:type nrg_model: EnergyModel
:param energy_est_threshold_pct: Allowed margin for estimated vs
optimal task placement energy cost
:type energy_est_threshold_pct: int
Compute optimal energy consumption (energy-optimal task placement)
and compare to energy consumption estimated from the trace.
Check that the estimated energy does not exceed the optimal energy by
more than ``energy_est_threshold_pct``` percents.
"""
nrg_model = nrg_model or self.nrg_model
exp_power = self._get_expected_power_df(nrg_model, capacity_margin_pct)
est_power = self._get_estimated_power_df(nrg_model)
exp_energy = series_integrate(exp_power.sum(axis=1), method='rect')
est_energy = series_integrate(est_power.sum(axis=1), method='rect')
msg = f'Estimated {est_energy} bogo-Joules to run workload, expected {exp_energy}'
threshold = exp_energy * (1 + (energy_est_threshold_pct / 100))
passed = est_energy < threshold
res = ResultBundle.from_bool(passed)
res.add_metric("estimated energy", est_energy, 'bogo-joules')
res.add_metric("energy threshold", threshold, 'bogo-joules')
return res
def _test_correctness(self, signal_name, mean_error_margin_pct,
max_error_margin_pct):
task = self.task_name
df = self.get_simulated_pelt(task, signal_name)
abs_error = df['error'].abs()
mean_error_pct = series_mean(abs_error) / UTIL_SCALE * 100
max_error_pct = abs_error.max() / UTIL_SCALE * 100
mean_ok = mean_error_pct <= mean_error_margin_pct
max_ok = max_error_pct <= max_error_margin_pct
res = ResultBundle.from_bool(mean_ok and max_ok)
res.add_metric('actual mean', series_mean(df[signal_name]))
res.add_metric('simulated mean', series_mean(df['simulated']))
res.add_metric('mean error', mean_error_pct, '%')
res.add_metric('actual max', df[signal_name].max())
res.add_metric('simulated max', df['simulated'].max())
res.add_metric('max error', max_error_pct, '%')
self._plot_pelt(task, signal_name, df['simulated'], 'correctness')
res = self._add_cpu_metric(res)
return res
def test_slack(self, negative_slack_allowed_pct=15) -> ResultBundle:
"""
Assert that the RTApp workload was given enough performance
:param negative_slack_allowed_pct: Allowed percentage of RT-app task
activations with negative slack.
:type negative_slack_allowed_pct: int
Use :class:`lisa.analysis.rta.RTAEventsAnalysis` to find instances
where the RT-App workload wasn't able to complete its activations (i.e.
its reported "slack" was negative). Assert that this happened less than
``negative_slack_allowed_pct`` percent of the time.
"""
self._check_valid_placement()
passed = True
bad_activations = {}
test_tasks = list(chain.from_iterable(self.rtapp_tasks_map.values()))
for task in test_tasks:
slack = self.trace.ana.rta.df_rtapp_stats(task)["slack"]
bad_activations_pct = len(slack[slack < 0]) * 100 / len(slack)
if bad_activations_pct > negative_slack_allowed_pct:
passed = False
bad_activations[task] = bad_activations_pct
res = ResultBundle.from_bool(passed)
for task, bad_activations_pct in bad_activations.items():
res.add_metric(f"{task} delayed activations", bad_activations_pct,
'%')
return res
def _test_cpus_busy(self, task_state_dfs, cpus, allowed_idle_time_s):
"""
Test that for every window in which the tasks are running, :attr:`cpus`
are not idle for more than :attr:`allowed_idle_time_s`
"""
if allowed_idle_time_s is None:
allowed_idle_time_s = 1e-3 * self.plat_info["cpus-count"]
res = ResultBundle.from_bool(True)
for task, state_df in task_state_dfs.items():
# Have a look at every task activation
task_idle_times = [
self._max_idle_time(index, index + row.delta, cpus)
for index, row in state_df.iterrows()
]
if not task_idle_times:
continue
max_time, max_cpu = max(task_idle_times)
res.add_metric("{} max idle".format(task),
data={
"time": TestMetric(max_time, "seconds"),
"cpu": TestMetric(max_cpu)
})
if max_time > allowed_idle_time_s:
res.result = Result.FAILED
return res
def test_slack(self, negative_slack_allowed_pct=15) -> ResultBundle:
"""
Assert that the RTApp workload was given enough performance
:param negative_slack_allowed_pct: Allowed percentage of RT-app task
activations with negative slack.
:type negative_slack_allowed_pct: int
Use :class:`lisa.analysis.rta.PerfAnalysis` to find instances where the
RT-App workload wasn't able to complete its activations (i.e. its
reported "slack" was negative). Assert that this happened less than
``negative_slack_allowed_pct`` percent of the time.
"""
analysis = PerfAnalysis.from_dir(self.res_dir)
passed = True
bad_activations = {}
for task in analysis.tasks:
slack = analysis.get_df(task)["Slack"]
bad_activations_pct = len(slack[slack < 0]) * 100 / len(slack)
if bad_activations_pct > negative_slack_allowed_pct:
passed = False
bad_activations[task] = bad_activations_pct
res = ResultBundle.from_bool(passed)
for task, bad_activations_pct in bad_activations.items():
res.add_metric("{} delayed activations".format(task),
bad_activations_pct, '%')
return res
def test_stune_task_placement(self, bad_cpu_margin_pct=10) -> ResultBundle:
"""
Test that the task placement satisfied the boost requirement
Check that top-app tasks spend no more than ``bad_cpu_margin_pct`` of
their time on CPUs that don't have enough capacity to serve their
boost.
"""
assert len(self.rtapp_tasks) == 1
task = self.rtapp_tasks[0]
df = self.trace.analysis.tasks.df_task_total_residency(task)
# Find CPUs without enough capacity to meet the boost
boost = self.boost
cpu_caps = self.plat_info['cpu-capacities']
ko_cpus = list(
filter(lambda x: (cpu_caps[x] / 10.24) < boost, cpu_caps))
# Count how much time was spend on wrong CPUs
time_ko = 0
total_time = 0
for cpu in cpu_caps:
t = df['runtime'][cpu]
if cpu in ko_cpus:
time_ko += t
total_time += t
pct_ko = time_ko * 100 / total_time
res = ResultBundle.from_bool(pct_ko < bad_cpu_margin_pct)
res.add_metric("time spent on inappropriate CPUs", pct_ko, '%')
res.add_metric("boost", boost, '%')
return res
def _test_cpus_busy(self, task_state_dfs, cpus, allowed_idle_time_s):
"""
Test that for every window in which the tasks are running, :attr:`cpus`
are not idle for more than :attr:`allowed_idle_time_s`
"""
if allowed_idle_time_s is None:
# Regular interval is 1 ms * nr_cpus, rounded to closest jiffy multiple
jiffy = 1 / self.plat_info['kernel']['config']['CONFIG_HZ']
interval = 1e-3 * self.plat_info["cpus-count"]
allowed_idle_time_s = ceil(interval / jiffy) * jiffy
res = ResultBundle.from_bool(True)
for task, state_df in task_state_dfs.items():
# Have a look at every task activation
task_idle_times = [self._max_idle_time(index, index + row.delta, cpus)
for index, row in state_df.iterrows()]
if not task_idle_times:
continue
max_time, max_cpu = max(task_idle_times)
res.add_metric("{} max idle".format(task), data={
"time": TestMetric(max_time, "seconds"), "cpu": TestMetric(max_cpu)})
if max_time > allowed_idle_time_s:
res.result = Result.FAILED
return res
def _test_behaviour(self, signal_name, error_margin_pct):
task = self.task_name
phase = self.wlgen_task.phases[0]
df = self.get_simulated_pelt(task, signal_name)
cpus = sorted(phase['cpus'])
assert len(cpus) == 1
cpu = cpus[0]
expected_duty_cycle_pct = phase['wload'].unscaled_duty_cycle_pct(self.plat_info)
expected_final_util = expected_duty_cycle_pct / 100 * UTIL_SCALE
settling_time = pelt_settling_time(10, init=0, final=expected_final_util)
settling_time += df.index[0]
df = df[settling_time:]
# Instead of taking the mean, take the average between the min and max
# values of the settled signal. This avoids the bias introduced by the
# fact that the util signal stays high while the task sleeps
settled_signal_mean = kernel_util_mean(df[signal_name], plat_info=self.plat_info)
expected_signal_mean = expected_final_util
signal_mean_error_pct = abs(expected_signal_mean - settled_signal_mean) / UTIL_SCALE * 100
res = ResultBundle.from_bool(signal_mean_error_pct < error_margin_pct)
res.add_metric('expected mean', expected_signal_mean)
res.add_metric('settled mean', settled_signal_mean)
res.add_metric('settled mean error', signal_mean_error_pct, '%')
self._plot_pelt(task, signal_name, df['simulated'], 'behaviour')
res = self._add_cpu_metric(res)
return res
def exp_power(row):
task_utils = row.to_dict()
try:
expected_utils = nrg_model.get_optimal_placements(
task_utils, capacity_margin_pct)[0]
except EnergyModelCapacityError:
ResultBundle.raise_skip(
'The workload will result in overutilized status for all possible task placement, making it unsuitable to test EAS on this platform'
)
power = nrg_model.estimate_from_cpu_util(expected_utils)
columns = list(power.keys())
# Assemble a dataframe to plot the expected utilization
data.append(expected_utils)
index.append(row.name)
return pd.Series([power[c] for c in columns], index=columns)
def test_cpus_alive(self) -> ResultBundle:
"""
Test that all CPUs came back online after the hotplug operations
"""
res = ResultBundle.from_bool(self.hotpluggable_cpus == self.live_cpus)
res.add_metric("hotpluggable CPUs", self.hotpluggable_cpus)
res.add_metric("Online CPUs", self.live_cpus)
return res
def test_output(self):
passed = False
for line in self.shell_output:
if '42' in line:
passed = True
break
return ResultBundle.from_bool(passed)
def test_cpus_alive(self) -> ResultBundle:
"""
Test that all CPUs came back online after the hotplug operations
"""
res = ResultBundle.from_bool(self.hotpluggable_cpus == self.live_cpus)
dead_cpus = sorted(set(self.hotpluggable_cpus) - set(self.live_cpus))
res.add_metric("dead CPUs", dead_cpus)
res.add_metric("number of dead CPUs", len(dead_cpus))
return res
def test_util_task_migration(self, allowed_error_pct=3) -> ResultBundle:
"""
Test that a migrated task properly propagates its utilization at the CPU level
:param allowed_error_pct: How much the trace averages can stray from the
expected values
:type allowed_error_pct: float
"""
expected_util = self.get_expected_cpu_util()
trace_util = self.get_trace_cpu_util()
passed = True
expected_metrics = {}
trace_metrics = {}
deltas = {}
for cpu in self.cpus:
expected_cpu_util = expected_util[cpu]
trace_cpu_util = trace_util[cpu]
cpu_str = f"cpu{cpu}"
expected_metrics[cpu_str] = TestMetric({})
trace_metrics[cpu_str] = TestMetric({})
deltas[cpu_str] = TestMetric({})
for phase in sorted(trace_cpu_util.keys()
& expected_cpu_util.keys()):
# TODO: remove that once we have named phases to skip the buffer phase
if phase == 0:
continue
expected_phase_util = expected_cpu_util[phase]
trace_phase_util = trace_cpu_util[phase]
is_equal, delta = self.is_almost_equal(expected_phase_util,
trace_phase_util,
allowed_error_pct)
if not is_equal:
passed = False
# Just some verbose metric collection...
phase_str = f"phase{phase}"
expected_metrics[cpu_str].data[phase_str] = TestMetric(
expected_phase_util)
trace_metrics[cpu_str].data[phase_str] = TestMetric(
trace_phase_util)
deltas[cpu_str].data[phase_str] = TestMetric(delta, "%")
res = ResultBundle.from_bool(passed)
res.add_metric("Expected utilization", expected_metrics)
res.add_metric("Trace utilization", trace_metrics)
res.add_metric("Utilization deltas", deltas)
self._plot_util()
return res
def test_placement(self) -> ResultBundle:
"""
For each phase, checks if the task placement is compatible with
UtilClamp requirements. This is done by comparing the maximum capacity
of the CPU on which the task has been placed, with the UtilClamp
value.
"""
metrics = {}
test_failures = []
capacity_margin = self.CAPACITY_MARGIN
cpu_max_capacities = self.plat_info['cpu-capacities']['rtapp']
def parse_phase(df, phase):
uclamp_val = phase['uclamp_val']
num_activations = df['active'][df['active'] == 1].count()
cpus = set(df.cpu.dropna().unique())
fitting_cpus = {
cpu
for cpu, cap in cpu_max_capacities.items()
if (cap == PELT_SCALE) or (cap * capacity_margin) > uclamp_val
}
failures = df[(df['active'] == 1)
& (df['cpu'].isin(cpus -
fitting_cpus))].index.tolist()
num_failures = len(failures)
test_failures.extend(failures)
phase_str = f"Phase-{phase['phase']}"
metrics[phase_str] = {
'uclamp-min':
TestMetric(uclamp_val),
'cpu-placements':
TestMetric(cpus),
'expected-cpus':
TestMetric(fitting_cpus),
'bad-activations':
TestMetric(num_failures * 100 / num_activations, "%"),
}
return cpus.issubset(fitting_cpus)
res = ResultBundle.from_bool(self._for_each_phase(parse_phase).all())
res.add_metric('Phases', metrics)
self._plot_phases('test_placement', test_failures)
return res
def test_util_task_migration(self, allowed_error_pct=5) -> ResultBundle:
"""
Test that a migrated task properly propagates its utilization at the CPU level
:param allowed_error_pct: How much the trace averages can stray from the
expected values
:type allowed_error_pct: float
"""
expected_cpu_util = self.get_expected_cpu_util()
trace_cpu_util = self.get_trace_cpu_util()
passed = True
expected_metrics = {}
trace_metrics = {}
deltas = {}
for cpu in self.cpus:
cpu_str = "cpu{}".format(cpu)
expected_metrics[cpu_str] = TestMetric({})
trace_metrics[cpu_str] = TestMetric({})
deltas[cpu_str] = TestMetric({})
for phase in range(self.nr_phases):
if not self.is_almost_equal(trace_cpu_util[cpu][phase],
expected_cpu_util[cpu][phase],
allowed_error_pct):
passed = False
# Just some verbose metric collection...
phase_str = "phase{}".format(phase)
expected = expected_cpu_util[cpu][phase]
trace = trace_cpu_util[cpu][phase]
delta = 100 * (trace - expected) / expected
expected_metrics[cpu_str].data[phase_str] = TestMetric(
expected)
trace_metrics[cpu_str].data[phase_str] = TestMetric(trace)
deltas[cpu_str].data[phase_str] = TestMetric(delta, "%")
res = ResultBundle.from_bool(passed)
res.add_metric("Expected utilization", expected_metrics)
res.add_metric("Trace utilization", trace_metrics)
res.add_metric("Utilization deltas", deltas)
return res
def test_performance_sanity(self) -> ResultBundle:
"""
Assert that higher CPU frequency leads to more work done
"""
res = ResultBundle.from_bool(True)
for cpu, freq_work in self.cpu_work.items():
sorted_freqs = sorted(freq_work.keys())
work = [freq_work[freq] for freq in sorted_freqs]
if not work == sorted(work):
res.result = Result.FAILED
res.add_metric("CPU{} work".format(cpu), freq_work)
return res
def test_capacity_sanity(self) -> ResultBundle:
"""
Assert that higher CPU capacity means more work done
"""
sorted_capacities = sorted(self.capacity_work.keys())
work = [self.capacity_work[cap] for cap in sorted_capacities]
# Check the list of work units is monotonically increasing
work_increasing = (work == sorted(work))
res = ResultBundle.from_bool(work_increasing)
capa_score = {}
for capacity, work in self.capacity_work.items():
capa_score[capacity] = TestMetric(work)
res.add_metric("Capacity to performance", capa_score)
return res
def test_task_remains(self) -> ResultBundle:
"""
Test that task remains on the same core
"""
test_passed = True
metrics = {}
for task_id in self.rtapp_task_ids:
cpu_df = self._get_task_cpu_df(task_id)
core_migrations = len(cpu_df.index)
metrics[task_id] = TestMetric(core_migrations)
# Ideally, task with 50% utilization
# should stay on the same core
if core_migrations > 1:
test_passed = False
res = ResultBundle.from_bool(test_passed)
res.add_metric("Migrations", metrics)
return res
def test_preempt_time(self, allowed_preempt_pct=1) -> ResultBundle:
"""
Test that tasks are not being preempted too much
"""
sdf = self.trace.df_events('sched_switch')
task_state_dfs = {
task : self.trace.analysis.tasks.df_task_states(task)
for task in self.rtapp_tasks
}
res = ResultBundle.from_bool(True)
for task, state_df in task_state_dfs.items():
# The sched_switch dataframe where the misfit task
# is replaced by another misfit task
preempt_sdf = sdf[
(sdf.prev_comm == task) &
(sdf.next_comm.str.startswith(self.task_prefix))
]
state_df = self._trim_state_df(
state_df[
(state_df.index.isin(preempt_sdf.index)) &
# Ensure this is a preemption and not just the task ending
(state_df.curr_state == TaskState.TASK_INTERRUPTIBLE)
]
)
preempt_time = state_df.delta.sum()
preempt_pct = (preempt_time / self.duration) * 100
res.add_metric("{} preemption".format(task), {
"ratio" : TestMetric(preempt_pct, "%"),
"time" : TestMetric(preempt_time, "seconds")})
if preempt_pct > allowed_preempt_pct:
res.result = Result.FAILED
return res
def test_stune_frequency(self, freq_margin_pct=10) -> ResultBundle:
"""
Test that frequency selection followed the boost
:param: freq_margin_pct: Allowed margin between estimated and measured
average frequencies
:type freq_margin_pct: int
Compute the expected frequency given the boost level and compare to the
real average frequency from the trace.
Check that the difference between expected and measured frequencies is
no larger than ``freq_margin_pct``.
"""
kernel_version = self.plat_info['kernel']['version']
if kernel_version.parts[:2] < (4, 14):
self.get_logger().warning(
'This test requires the RT boost hold, but it may be disabled in {}'
.format(kernel_version))
cpu = self.plat_info['capacity-classes'][-1][0]
freqs = self.plat_info['freqs'][cpu]
max_freq = max(freqs)
# Estimate the target frequency, including sugov's margin, and round
# into a real OPP
boost = self.boost
target_freq = min(max_freq, max_freq * boost / 80)
target_freq = list(filter(lambda f: f >= target_freq, freqs))[0]
# Get the real average frequency
avg_freq = self.trace.analysis.frequency.get_average_cpu_frequency(cpu)
distance = abs(target_freq - avg_freq) * 100 / target_freq
res = ResultBundle.from_bool(distance < freq_margin_pct)
res.add_metric("target freq", target_freq, 'kHz')
res.add_metric("average freq", avg_freq, 'kHz')
res.add_metric("boost", boost, '%')
return res
def _test_signal(self, signal_name, allowed_error_pct):
passed = True
expected_data = {}
trace_data = {}
capacity = self._get_freq_capa(self.cpu, self.freq, self.plat_info)
for name in self.rtapp_tasks:
ok, exp_util, signal_mean = self._test_task_signal(
signal_name, allowed_error_pct, self.trace, self.cpu, name, capacity)
if not ok:
passed = False
expected_data[name] = TestMetric(exp_util)
trace_data[name] = TestMetric(signal_mean)
freq_str = '@{}'.format(self.freq) if self.freq is not None else ''
bundle = ResultBundle.from_bool(passed)
bundle.add_metric("cpu", '{}{}'.format(self.cpu, freq_str))
bundle.add_metric("Expected signals", expected_data)
bundle.add_metric("Trace signals", trace_data)
return bundle
def _test_range(self, signal_name, allowed_error_pct):
res = ResultBundle.from_bool(True)
task = self.rtapp_profile[self.task_name]
cpu = task.phases[0].cpus[0]
# Note: This test-case is only valid if executed at capacity == 1024.
# The below assertion is insufficient as it only checks the CPU can potentially
# reach a capacity of 1024.
assert self.plat_info["cpu-capacities"][cpu] == UTIL_SCALE
peltsim, pelt_task, sim_df = self.get_simulated_pelt(cpu, signal_name)
signal_df = self.get_task_sched_signal(cpu, signal_name)
sim_range = peltsim.stableRange(pelt_task)
# Get signal statistics in a period of time where the signal is
# supposed to be stable
signal_stats = signal_df[UTIL_AVG_CONVERGENCE_TIME_S:][
signal_name].describe()
expected_data = {}
trace_data = {}
for stat in ['min', 'max']:
stat_value = getattr(sim_range, '{}_value'.format(stat))
if not self.is_almost_equal(stat_value, signal_stats[stat],
allowed_error_pct):
res.result = Result.FAILED
trace_data[stat] = TestMetric(signal_stats[stat])
expected_data[stat] = TestMetric(stat_value)
res.add_metric("Trace signal", trace_data)
res.add_metric("Expected signal", expected_data)
return res
def check_from_target(cls, target):
super().check_from_target(target)
kconfig = target.plat_info['kernel']['config']
for option in (
'CONFIG_ENERGY_MODEL',
'CONFIG_CPU_FREQ_GOV_SCHEDUTIL',
):
if not kconfig.get(option):
ResultBundle.raise_skip(
f"The target's kernel needs {option}=y kconfig enabled")
for domain in target.plat_info['freq-domains']:
if "schedutil" not in target.cpufreq.list_governors(domain[0]):
ResultBundle.raise_skip(
f"Can't set schedutil governor for domain {domain}")
if 'nrg-model' not in target.plat_info:
ResultBundle.raise_skip("Energy model not available")
def get_simulated_pelt(self, task, signal_name):
"""
Simulate a PELT signal for a given task.
:param task: task to look for in the trace.
:type task: int or str or tuple(int, str)
:param signal_name: Name of the PELT signal to simulate.
:type signal_name: str
:return: A :class:`pandas.DataFrame` with a ``simulated`` column
containing the simulated signal, along with the column of the
signal as found in the trace.
"""
logger = self.logger
trace = self.trace
task = trace.get_task_id(task)
df_activation = trace.ana.tasks.df_task_activation(
task,
# Util only takes into account times where the task is actually
# executing
preempted_value=0,
)
pinned_cpus = sorted(self.cpus)
assert len(pinned_cpus) == 1
df = self._get_trace_signal(task, pinned_cpus, signal_name)
df = df.copy(deep=False)
# Ignore the first activation, as its signals are incorrect
df_activation = df_activation.iloc[2:]
# Make sure the activation df does not start before the dataframe of
# signal values, otherwise we cannot provide a sensible init value
df_activation = df_activation[df.index[0]:]
# Get the initial signal value matching the first activation we will care about
init_iloc = df.index.get_indexer([df_activation.index[0]],
method='ffill')[0]
init = df[signal_name].iloc[init_iloc]
try:
# PELT clock in nanoseconds
clock = df['update_time'] * 1e-9
except KeyError:
if any(self.plat_info['cpu-capacities']['rtapp'][cpu] != UTIL_SCALE
for phase in self.wlgen_task.phases
for cpu in phase['cpus']):
ResultBundle.raise_skip(
'PELT time scaling can only be simulated when the PELT clock is available from the trace'
)
logger.warning(
'PELT clock is not available, ftrace timestamp will be used at the expense of accuracy'
)
clock = None
try:
cpus = trace.ana.tasks.cpus_of_tasks([task])
capacity = trace.ana.load_tracking.df_cpus_signal('capacity', cpus)
except MissingTraceEventError:
capacity = None
else:
capacity = capacity[['cpu', 'capacity_curr']]
# We are interested in the current CPU capacity as seen by CFS.
# This takes into account:
# * The frequency
# * The capacity of other sched classes (RT, IRQ etc)
capacity = capacity.rename(columns={'capacity_curr': 'capacity'})
# Reshape the capacity dataframe so that we get one column per CPU
capacity = capacity.pivot(columns=['cpu'])
capacity.columns = capacity.columns.droplevel(0)
capacity.ffill(inplace=True)
capacity = df_refit_index(capacity,
window=(df_activation.index[0],
df_activation.index[-1]))
# Make sure we end up with the timestamp at which the capacity
# changes, rather than the timestamps at which the task is enqueued
# or dequeued.
activation_cpu = df_activation['cpu'].reindex(capacity.index,
method='ffill')
capacity = series_dereference(activation_cpu, capacity)
df['simulated'] = simulate_pelt(
df_activation['active'],
index=df.index,
init=init,
clock=clock,
capacity=capacity,
)
# Since load is now CPU invariant in recent kernel versions, we don't
# rescale it back. To match the old behavior, that line is
# needed:
# df['simulated'] /= self.plat_info['cpu-capacities']['rtapp'][cpu] / UTIL_SCALE
kernel_version = self.plat_info['kernel']['version']
if (signal_name == 'load' and kernel_version.parts[:2] < (5, 1)):
logger().warning(
f'Load signal is assumed to be CPU invariant, which is true for recent mainline kernels, but may be wrong for {kernel_version}'
)
df['error'] = df[signal_name] - df['simulated']
df = df.dropna()
return df
def test_activations(self) -> ResultBundle:
"""
Test signals are properly "aggregated" at enqueue/dequeue time.
On fast-ramp systems, `util_est_enqueud` is expected to be always
smaller than `util_est_ewma`.
On non fast-ramp systems, the `util_est_enqueued` is expected to be
smaller then `util_est_ewma` in ramp-down phases, or bigger in ramp-up
phases.
Those conditions are checked on a single execution of a task which has
three main behaviours:
* STABLE: periodic big task running for a relatively long period to
ensure `util_avg` saturation.
* DOWN: periodic ramp-down task, to slowly decay `util_avg`
* UP: periodic ramp-up task, to slowly increase `util_avg`
"""
failure_reasons = {}
metrics = {}
# We have only two task: the main 'rt-app' task and our 'test_task'
test_task = self.trace.analysis.rta.rtapp_tasks[-1]
# Get list of task's activations
df = self.trace.analysis.tasks.df_task_states(test_task)
activations = df[(df.curr_state == TaskState.TASK_WAKING)
& (df.next_state == TaskState.TASK_ACTIVE)].index
# Check task signals at each activation
df = self.trace.df_events('sched_util_est_task')
df = df_filter_task_ids(df, [test_task])
# Define a time interval to correlate relative trace events.
def restrict(df, time, delta=1e-3):
return df[time - delta:time + delta]
failures = []
for idx, activation in enumerate(activations):
avg, enq, ewma = restrict(df, activation)[[
'util_avg', 'util_est_enqueued', 'util_est_ewma'
]].iloc[-1]
metrics[idx + 1] = ActivationSignals(activation, avg, enq, ewma)
# UtilEst is not updated when within 1% of previous activation
if 1.01 * enq < avg:
failure_reasons[idx] = 'enqueued({}) smaller than util_avg({}) @ {}'\
.format(enq, avg, activation)
failures.append(activation)
continue
# Running on FastRamp kernels:
if self.fast_ramp:
# STABLE, DOWN and UP:
if enq > ewma:
failure_reasons[idx] = 'enqueued({}) bigger than ewma({}) @ {}'\
.format(enq, ewma, activation)
failures.append(activation)
continue
# Running on (legacy) non FastRamp kernels:
else:
phase = self.trace.analysis.rta.task_phase_at(
test_task, activation)
# STABLE: ewma ramping up
if phase.id == 0 and enq < ewma:
failure_reasons[idx] = 'enqueued({}) smaller than ewma({}) @ {}'\
.format(enq, ewma, activation)
failures.append(activation)
continue
# DOWN: ewma ramping down
if 0 < phase.id < 5 and enq > ewma:
failure_reasons[idx] = 'enqueued({}) bigger than ewma({}) @ {}'\
.format(enq, ewma, activation)
failures.append(activation)
continue
# UP: ewma ramping up
if phase.id > 4 and enq < ewma:
failure_reasons[idx] = 'enqueued({}) smaller than ewma({}) @ {}'\
.format(enq, ewma, activation)
failures.append(activation)
continue
self._plot_signals(test_task, 'activations', failures)
bundle = ResultBundle.from_bool(not failure_reasons)
bundle.add_metric("signals", metrics)
bundle.add_metric("failure reasons", failure_reasons)
return bundle
