def test_performance_sanity(self) -> ResultBundle:
"""
Assert that higher CPU frequency leads to more work done
"""
res = ResultBundle.from_bool(True)
cpu_items = {
cpu: {
# We expect only one item per frequency
item.freq: item
for item in freq_items
}
for cpu, freq_items in groupby(self.sanity_items, key=lambda item: item.cpu)
}
failed = []
passed = True
for cpu, freq_items in cpu_items.items():
sorted_items = sorted(freq_items.values(), key=lambda item: item.freq)
work = [item.work for item in sorted_items]
if work != sorted(work):
passed = False
failed.append(cpu)
res = ResultBundle.from_bool(passed)
work_metric = {
cpu: {freq: item.work for freq, item in freq_items.items()}
for cpu, freq_items in cpu_items.items()
}
res.add_metric('CPUs work', work_metric)
res.add_metric('Failed CPUs', failed)
return res
def test_cpu_invariance(self) -> AggregatedResultBundle:
"""
Check that items using the max freq on each CPU is passing util avg test.
There could be false positives, but they are expected to be relatively
rare.
.. seealso:: :class:`TaskInvariance.ITEM_CLS.test_util_behaviour`
"""
res_list = []
for cpu, item_group in groupby(self.invariance_items,
key=lambda x: x.cpu):
item_group = list(item_group)
# combine all frequencies of that CPU class, although they should
# all be the same
max_freq = max(
itertools.chain.from_iterable(x.freq_list for x in item_group))
max_freq_items = [
item for item in item_group if item.freq == max_freq
]
for item in max_freq_items:
# Only test util, as it should be more robust
res = item.test_util_behaviour()
res_list.append(res)
return AggregatedResultBundle(res_list, 'cpu')
def get_prebuilt_op_set(self):
non_reusable_type_set = self.get_non_reusable_type_set()
op_set = set()
# Try to build as many configurations instances from all the files we
# are given
conf_cls_set = set(get_subclasses(MultiSrcConf, only_leaves=True))
conf_list = []
for conf_path in self.args.conf:
for conf_cls in conf_cls_set:
try:
# Do not add the default source, to avoid overriding user
# configuration with the default one.
conf = conf_cls.from_yaml_map(conf_path,
add_default_src=False)
except ValueError:
continue
else:
conf_list.append((conf, conf_path))
def keyfunc(conf_and_path):
cls = type(conf_and_path[0])
# We use the ID since classes are not comparable
return id(cls), cls
# Then aggregate all the conf from each type, so they just act as
# alternative sources.
for (_, conf_cls), conf_and_path_seq in groupby(conf_list,
key=keyfunc):
conf_and_path_list = list(conf_and_path_seq)
# Get the default configuration, and stack all user-defined keys
conf = conf_cls()
for conf_src, conf_path in conf_and_path_list:
src = os.path.basename(conf_path)
conf.add_src(src, conf_src)
op_set.add(
PrebuiltOperator(conf_cls, [conf],
non_reusable_type_set=non_reusable_type_set))
# Inject serialized objects as root operators
for path in self.args.inject:
obj = Serializable.from_path(path)
op_set.add(
PrebuiltOperator(type(obj), [obj],
non_reusable_type_set=non_reusable_type_set))
# Inject a dummy empty TargetConf
if self.args.inject_empty_target_conf:
op_set.add(
PrebuiltOperator(TargetConf, [TargetConf(conf={})],
non_reusable_type_set=non_reusable_type_set))
return op_set
def test_freq_invariance(self) -> ResultBundle:
"""
Check that at least one CPU has items passing for all tested frequencies.
.. seealso:: :class:`InvarianceItem.test_task_util_avg`
"""
logger = self.get_logger()
def make_group_bundle(cpu, item_group):
bundle = AggregatedResultBundle(
[
# Only test util, as it should be more robust
item.test_task_util_avg() for item in item_group
],
# each item's "cpu" metric also contains the frequency
name_metric='cpu',
)
# At that level, we only report the CPU, since nested bundles cover
# different frequencies
bundle.add_metric('cpu', cpu)
logger.info('Util avg invariance {res} for CPU {cpu}'.format(
res=bundle.result.lower_name,
cpu=cpu,
))
return bundle
group_result_bundles = [
make_group_bundle(cpu, item_group)
for cpu, item_group in groupby(self.invariance_items,
key=lambda x: x.cpu)
]
# The combination differs from the AggregatedResultBundle default one:
# we consider as passed as long as at least one of the group has
# passed, instead of forcing all of them to pass.
if any(result_bundle.result is Result.PASSED
for result_bundle in group_result_bundles):
overall_result = Result.PASSED
elif all(result_bundle.result is Result.UNDECIDED
for result_bundle in group_result_bundles):
overall_result = Result.UNDECIDED
else:
overall_result = Result.FAILED
return AggregatedResultBundle(group_result_bundles,
name_metric='cpu',
result=overall_result)
def group_by_testcase(froz_val_list):
return OrderedDict(
(testcase_id, [froz_val.value for froz_val in froz_val_group])
for testcase_id, froz_val_group in groupby(froz_val_list,
key=get_id))
def make_changelog(repo):
"""
Generate a reStructuredText changelog to be included in the documentation.
.. note:: The git repository cannot be a shallow clone, as the changelog is
extracted from the git history.
"""
release_refs = ['HEAD'] + lisa._git.find_tags(repo, 'v*')
def update_release_name(name):
if name == 'HEAD':
return 'Next release'
else:
return name
MARKERS = ['FEATURE', 'FIX', 'BREAKING CHANGE']
# Filtering on the patterns we look for provides a considerable speedup
commit_pattern = '(' + '|'.join(map(re.escape, MARKERS)) + ')'
release_sha1s = {
update_release_name(y): lisa._git.find_commits(
repo=repo,
ref=f'{x}..{y}',
grep=commit_pattern,
regex=True,
)
for x, y in zip(release_refs[1:], release_refs)
}
release_msgs = {
release: [
lisa._git.get_commit_message(
repo=repo,
ref=ref,
format='%B',
).strip() for ref in refs
]
for release, refs in release_sha1s.items()
}
def parse_msg(msg):
selected = tuple(
sorted({marker
for marker in MARKERS if marker in msg}))
for marker in selected:
pattern = f'^\s*{re.escape(marker)}\s*$'
msg = re.sub(pattern, '', msg, flags=re.MULTILINE)
return (msg, selected)
def expand(msg, markers):
for marker in markers:
yield (marker, msg)
release_msgs = {
release: dict(
map(
lambda x: (x[0], list(map(itemgetter(1), x[1]))),
groupby((entry for msg in msgs
for entry in expand(*parse_msg(msg))),
key=itemgetter(0))))
for release, msgs in release_msgs.items()
}
def indent(level, content):
idt = level * ' '
return idt + content.replace('\n', f'\n{idt}')
def format_release(name, sections):
title = f'{name}\n{len(name) * "="}\n'
body = '\n\n'.join(
format_section(marker, _msgs) for marker, _msgs in order_as(
sections.items(),
order_as=MARKERS,
key=itemgetter(0),
))
return f'{title}\n{body}'
def format_section(name, msgs):
title = f'{name.capitalize()}\n{len(name) * "+"}\n'
body = '\n\n'.join(map(format_msg, sorted(msgs)))
body = indent(4, body)
return f'{title}\n{body}'
def format_msg(msg):
subject = msg.splitlines()[0]
return f'- {subject.strip()}'
rst = '\n\n'.join(
format_release(name, sections)
for name, sections in release_msgs.items())
return rst
def get_deprecated_table():
"""
Get a reStructuredText tables with titles for all the deprecated names in
:mod:`lisa`.
"""
def indent(string, level=1):
idt = ' ' * 4
return string.replace('\n', '\n' + idt * level)
def make_entry(entry):
msg = entry.get('msg') or ''
removed_in = entry.get('removed_in')
if removed_in is None:
removed_in = ''
else:
removed_in = f'*Removed in: {format_version(removed_in)}*\n\n'
name = get_sphinx_name(entry['obj'], style='rst')
replaced_by = entry.get('replaced_by')
if replaced_by is None:
replaced_by = ''
else:
replaced_by = f"*Replaced by:* {get_sphinx_name(replaced_by, style='rst')}\n\n"
return "* - {name}{msg}{replaced_by}{removed_in}".format(
name=indent(name + '\n\n'),
msg=indent(msg + '\n\n' if msg else ''),
replaced_by=indent(replaced_by),
removed_in=indent(removed_in),
)
def make_table(entries, removed_in):
if entries:
entries = '\n'.join(
make_entry(entry)
for entry in sorted(entries, key=itemgetter('name')))
if removed_in:
if removed_in > lisa.version.version_tuple:
remove = 'to be removed'
else:
remove = 'removed'
removed_in = f' {remove} in {format_version(removed_in)}'
else:
removed_in = ''
table = ".. list-table:: Deprecated names{removed_in}\n :align: left{entries}".format(
entries=indent('\n\n' + entries),
removed_in=removed_in,
)
header = f'Deprecated names{removed_in}'
header += '\n' + '+' * len(header)
return header + '\n\n' + table
else:
return ''
entries = [{
'name': name,
**info
} for name, info in get_deprecated_map().items()]
unspecified_removal = [
entry for entry in entries if not entry['removed_in']
]
other_entries = [
entry for entry in entries if entry not in unspecified_removal
]
tables = []
tables.append(make_table(unspecified_removal, removed_in=None))
tables.extend(
make_table(entries, removed_in=removed_in) for removed_in, entries in
groupby(other_entries, itemgetter('removed_in'), reverse=True))
return '\n\n'.join(tables)
def _calibrate(cls, target, res_dir):
res_dir = res_dir if res_dir else target .get_res_dir(
"rta_calib", symlink=False
)
pload_regexp = re.compile(r'pLoad = ([0-9]+)ns')
pload = {}
logger = cls.get_logger()
# Create calibration task
if target.is_rooted:
max_rtprio = int(target.execute('ulimit -Hr').split('\r')[0])
logger.debug('Max RT prio: {}'.format(max_rtprio))
priority = max_rtprio if max_rtprio <= 10 else 10
sched_policy = 'FIFO'
else:
logger.warning('Will use default scheduler class instead of RT since the target is not rooted')
priority = None
sched_policy = None
for cpu in target.list_online_cpus():
logger.info('CPU{} calibration...'.format(cpu))
# RT-app will run a calibration for us, so we just need to
# run a dummy task and read the output
calib_task = Periodic(
duty_cycle_pct=100,
duration_s=0.001,
period_ms=1,
priority=priority,
sched_policy=sched_policy,
)
rta = cls.by_profile(target, name="rta_calib_cpu{}".format(cpu),
profile={'task1': calib_task},
calibration="CPU{}".format(cpu),
res_dir=res_dir)
with rta, target.freeze_userspace():
# Disable CPU capacities update, since that leads to infinite
# recursion
rta.run(as_root=target.is_rooted, update_cpu_capacities=False)
for line in rta.output.split('\n'):
pload_match = re.search(pload_regexp, line)
if pload_match is None:
continue
pload[cpu] = int(pload_match.group(1))
logger.debug('>>> CPU{}: {}'.format(cpu, pload[cpu]))
# Avoid circular import issue
from lisa.platforms.platinfo import PlatformInfo
snippet_plat_info = PlatformInfo({
'rtapp': {
'calib': pload,
},
})
logger.info('Platform info rt-app calibration configuration:\n{}'.format(
snippet_plat_info.to_yaml_map_str()
))
plat_info = target.plat_info
# Sanity check calibration values for asymmetric systems if we have
# access to capacities
try:
cpu_capacities = plat_info['cpu-capacities']
except KeyError:
return pload
capa_ploads = {
capacity: {cpu: pload[cpu] for cpu, capa in cpu_caps}
for capacity, cpu_caps in groupby(cpu_capacities.items(), itemgetter(1))
}
# Find the min pload per capacity level, i.e. the fastest detected CPU.
# It is more likely to represent the right pload, as it has suffered
# from less IRQ slowdown or similar disturbances that might be random.
capa_pload = {
capacity: min(ploads.values())
for capacity, ploads in capa_ploads.items()
}
# Sort by capacity
capa_pload_list = sorted(capa_pload.items())
# unzip the list of tuples
_, pload_list = zip(*capa_pload_list)
# If sorting according to capa was not equivalent to reverse sorting
# according to pload (small pload=fast cpu)
if list(pload_list) != sorted(pload_list, reverse=True):
raise CalibrationError('Calibration values reports big cores less capable than LITTLE cores')
# Check that the CPU capacities seen by rt-app are similar to the one
# the kernel uses
true_capacities = cls.get_cpu_capacities_from_calibrations(pload)
capa_factors_pct = {
cpu: true_capacities[cpu] / cpu_capacities[cpu] * 100
for cpu in cpu_capacities.keys()
}
dispersion_pct = max(abs(100 - factor) for factor in capa_factors_pct.values())
logger.info('CPU capacities according to rt-app workload: {}'.format(true_capacities))
if dispersion_pct > 2:
logger.warning('The calibration values are not inversely proportional to the CPU capacities, the duty cycles will be up to {:.2f}% off on some CPUs: {}'.format(dispersion_pct, capa_factors_pct))
if dispersion_pct > 20:
logger.warning('The calibration values are not inversely proportional to the CPU capacities. Either rt-app calibration failed, or the rt-app busy loops has a very different instruction mix compared to the workload used to establish the CPU capacities: {}'.format(capa_factors_pct))
# Map of CPUs X to list of CPUs Ys that are faster than it although CPUs
# of Ys have a smaller capacity than X
if len(capa_ploads) > 1:
faster_than_map = {
cpu1: sorted(
cpu2
for cpu2, pload2 in ploads2.items()
# CPU2 faster than CPU1
if pload2 < pload1
)
for (capa1, ploads1), (capa2, ploads2) in itertools.permutations(capa_ploads.items())
for cpu1, pload1 in ploads1.items()
# Only look at permutations in which CPUs of ploads1 are supposed
# to be faster than the one in ploads2
if capa1 > capa2
}
else:
faster_than_map = {}
# Remove empty lists
faster_than_map = {
cpu: faster_cpus
for cpu, faster_cpus in faster_than_map.items()
if faster_cpus
}
if faster_than_map:
raise CalibrationError('Some CPUs of higher capacities are slower than other CPUs of smaller capacities: {}'.format(faster_than_map))
return pload
SignalDesc('sched_wakeup_new', ['target_cpu']),
SignalDesc('sched_wakeup_new', ['comm', 'pid']),
SignalDesc('cpu_idle', ['cpu_id']),
SignalDesc('cpu_frequency', ['cpu']),
SignalDesc('sched_compute_energy', ['comm', 'pid']),
SignalDesc('sched_load_se', ['comm', 'pid']),
SignalDesc('sched_util_est_task', ['comm', 'pid']),
SignalDesc('sched_util_est_cpu', ['cpu']),
SignalDesc('sched_load_cfs_rq', ['path', 'cpu']),
SignalDesc('sched_pelt_irq', ['cpu']),
SignalDesc('sched_pelt_rt', ['cpu']),
SignalDesc('sched_pelt_dl', ['cpu']),
SignalDesc('uclamp_util_se', ['pid', 'comm']),
SignalDesc('uclamp_util_cfs', ['cpu']),
SignalDesc('sched_overutilized', []),
SignalDesc('sched_process_wait', ['comm', 'pid']),
SignalDesc('schedutil_em_boost', ['cpu']),
SignalDesc('thermal_temperature', ['id']),
SignalDesc('thermal_zone_trip', ['id']),
]
"""
List of predefined :class:`SignalDesc`.
"""
SignalDesc._SIGNALS_MAP = {
event: list(signal_descs)
for event, signal_descs in groupby(_SIGNALS, key=attrgetter('event'))
}
# vim :set tabstop=4 shiftwidth=4 textwidth=80 expandtab
