class TestConfWithDefault(MultiSrcConf):
STRUCTURE = TopLevelKeyDesc('lisa-self-test-test-conf-with-default',
'lisa self test', INTERNAL_STRUCTURE)
DEFAULT_SRC = {
'bar': [0, 1, 2],
}
class DocPlotConf(SimpleMultiSrcConf):
"""
Analysis plot method arguments configuration for the documentation.
{generated_help}
"""
STRUCTURE = TopLevelKeyDesc('doc-plot-conf', 'Plot methods configuration', (
KeyDesc('plots', 'Mapping of function qualnames to their settings', [Mapping]),
))
class Gem5EnergyMeterConf(SimpleMultiSrcConf, HideExekallID):
"""
Configuration class for :class:`Gem5EnergyMeter`.
{generated_help}
"""
STRUCTURE = TopLevelKeyDesc(
'gem5-energy-meter-conf', 'Gem5 Energy Meter configuration',
(KeyDesc('channel-map', 'Channels to use', [Mapping]), ))
class HWMonConf(SimpleMultiSrcConf, HideExekallID):
"""
Configuration class for :class:`HWMon`.
{generated_help}
"""
STRUCTURE = TopLevelKeyDesc('hwmon-conf', 'HWMon Energy Meter configuration', (
# TODO: find a better help and maybe a better type
KeyDesc('channel-map', 'Channels to use', [Mapping]),
))
class FtraceConf(SimpleMultiSrcConf, HideExekallID):
"""
Configuration class of :class:`FtraceCollector`
Available keys:
{generated_help}
"""
STRUCTURE = TopLevelKeyDesc('ftrace-conf', 'FTrace configuration', (
KeyDesc('events', 'FTrace events to trace', [StrList]),
KeyDesc('functions', 'FTrace functions to trace', [StrList]),
KeyDesc('buffer-size', 'FTrace buffer size', [int]),
))
def add_merged_src(self, src, conf, **kwargs):
"""
Merge-in a configuration source.
:param src: Name of the merged source
:type src: str
:param conf: Conf to merge in
:type conf: FtraceConf
"""
def merge_conf(key, val):
if key in ('events', 'functions'):
return sorted(set(val) | set(self.get(key, [])))
elif key == 'buffer-size':
return max(val, self.get(key, 0))
else:
raise KeyError('Cannot merge key "{}"'.format(key))
merged = {
key: merge_conf(key, val)
for key, val in conf.items()
}
def is_modified(key, val):
try:
existing_val = self[key]
except KeyError:
return True
else:
return val != existing_val
# We merge some keys with their current value in the conf
return self.add_src(src,
conf={
key: val
for key, val in merged.items()
# Only add to the source if the result is different than what is
# already set
if is_modified(key, val)
},
**kwargs,
)
class ACMEConf(SimpleMultiSrcConf, HideExekallID):
"""
Configuration class for :class:`ACME`.
{generated_help}
"""
STRUCTURE = TopLevelKeyDesc('acme-conf', 'ACME Energy Meter configuration', (
KeyDesc('channel-map', 'Channels to use', [Mapping]),
KeyDesc('host', 'Hostname or IP address of the ACME board', [str]),
KeyDesc('iio-capture-bin', 'path to iio-capture binary', [str]),
))
class MonsoonConf(SimpleMultiSrcConf, HideExekallID):
"""
Configuration class for :class:`Monsoon`.
{generated_help}
"""
STRUCTURE = TopLevelKeyDesc('monsoon-conf', 'Monsoon Energy Meter configuration', (
KeyDesc('channel-map', 'Channels to use', [Mapping]),
KeyDesc('monsoon-bin', 'monsoon binary path', [str]),
KeyDesc('tty-device', 'TTY device to use', [str]),
))
class DmesgTestConf(SimpleMultiSrcConf):
"""
Configuration class for :meth:`lisa.tests.base.DmesgTestBundle.test_dmesg`.
{generated_help}
"""
STRUCTURE = TopLevelKeyDesc(
'dmesg-test-conf', 'Dmesg test configuration', (KeyDesc(
'ignored-patterns',
'List of Python regex matching dmesg entries content to be whitelisted',
[StrList]), ))
class DocPlotConf(SimpleMultiSrcConf):
"""
Analysis plot method arguments configuration for the documentation.
{generated_help}
"""
STRUCTURE = TopLevelKeyDesc('doc-plot-conf', 'Plot methods configuration', (
# Avoid deepcopy of the value, since it contains a Trace object that we
# don't want to duplicate for speed reasons
KeyDesc('plots', 'Mapping of function qualnames to their settings', [Mapping], deepcopy_val=False),
))
class AEPConf(SimpleMultiSrcConf, HideExekallID):
"""
Configuration class for :class:`AEP`.
{generated_help}
"""
STRUCTURE = TopLevelKeyDesc('aep-conf', 'AEP Energy Meter configuration', (
KeyDesc('channel-map', 'Channels to use', [Mapping]),
KeyDesc('resistor-values', 'Resistor values', [FloatList]),
KeyDesc('labels', 'List of labels', [StrList]),
KeyDesc('device-entry', 'TTY device', [StrList]),
))
class TestConf(MultiSrcConf):
STRUCTURE = TopLevelKeyDesc('lisa-self-test-test-conf', 'lisa self test',
INTERNAL_STRUCTURE)
class PlatformInfo(MultiSrcConf, HideExekallID):
"""
Platform-specific information made available to tests.
{generated_help}
"""
# we could use mypy.subtypes.is_subtype and use the infrastructure provided
# by typing module, but adding an external dependency is overkill for what
# we need.
STRUCTURE = TopLevelKeyDesc('platform-info', 'Platform-specific information', (
LevelKeyDesc('rtapp', 'RTapp configuration', (KeyDesc(
'calib', 'RTapp calibration dictionary',
[TypedDict[int, int]]), )),
LevelKeyDesc('kernel', 'Kernel-related information', (
KeyDesc(
'version', '', [KernelVersion]),
KernelConfigKeyDesc('config', '', [TypedKernelConfig]),
KernelSymbolsAddress(
'symbols-address',
'Dictionary of addresses to symbol names extracted from /proc/kallsyms',
[TypedDict[int, str]],
deepcopy_val=False),
)),
KeyDesc('nrg-model', 'Energy model object', [EnergyModel]),
LevelKeyDesc('cpu-capacities', 'Dictionaries of CPU ID to capacity value', (
KeyDesc(
'writeable',
'Whether the CPU capacities can be updated by writing in sysfs on this platform',
[bool
]),
KeyDesc(
'orig',
'Default capacity value as exposed by the kernel',
[CPUCapacities]),
DerivedKeyDesc(
'rtapp',
'CPU capacities adjusted with rtapp calibration values, for accurate duty cycle reproduction',
[CPUCapacities],
[['..', 'rtapp', 'calib'], ['orig'], ['writeable']],
compute_rtapp_capacities,
),
)),
KeyDesc('abi', 'ABI, e.g. "arm64"', [str]),
KeyDesc('os', 'OS being used, e.g. "linux"', [str]),
KeyDesc('name', 'Free-form name of the board', [str]),
KeyDesc('cpus-count', 'Number of CPUs', [int]),
KeyDesc('numa-nodes-count', 'Number of NUMA nodes', [int]),
KeyDesc(
'freq-domains',
'Frequency domains modeled by a list of CPU IDs for each domain',
[TypedList[CPUIdList]]),
KeyDesc('freqs', 'Dictionnary of CPU ID to list of frequencies',
[TypedDict[int, FreqList]]),
DerivedKeyDesc(
'capacity-classes',
'Capacity classes modeled by a list of CPU IDs for each capacity, sorted by capacity',
[TypedList[CPUIdList]], [['cpu-capacities', 'orig']],
compute_capa_classes),
))
"""Some keys have a reserved meaning with an associated type."""
def add_target_src(self,
target,
rta_calib_res_dir,
src='target',
only_missing=True,
**kwargs):
"""
Add source from a live :class:`lisa.target.Target`.
:param target: Target to inspect.
:type target: lisa.target.Target
:param rta_calib_res_dir: Result directory for rt-app calibrations.
:type rta_calib_res_dir: str
:param src: Named of the added source.
:type src: str
:param only_missing: If ``True``, only add values for the keys that are
not already provided by another source. This allows speeding up the
connection to target, at the expense of not being able to spot
inconsistencies between user-provided values and autodetected values.
:type only_missing: bool
:Variable keyword arguments: Forwarded to
:class:`lisa.conf.MultiSrcConf.add_src`.
"""
info = {
'nrg-model': lambda: self._nrg_model_from_target(target),
'kernel': {
'version': lambda: target.kernel_version,
'config': lambda: target.config.typed_config,
},
'abi': lambda: target.abi,
'os': lambda: target.os,
'rtapp': {
# Since it is expensive to compute, use an on-demand DeferredValue
'calib':
lambda: DeferredValue(RTA.get_cpu_calibrations, target,
rta_calib_res_dir)
},
'cpus-count': lambda: target.number_of_cpus,
'numa-nodes-count': lambda: target.number_of_nodes
}
def get_freq_domains():
if target.is_module_available('cpufreq'):
return list(target.cpufreq.iter_domains())
else:
return None
info['freq-domains'] = get_freq_domains
def get_freqs():
if target.is_module_available('cpufreq'):
freqs = {
cpu: target.cpufreq.list_frequencies(cpu)
for cpu in range(target.number_of_cpus)
}
# Only add the frequency info if there is any, otherwise don't
# mislead the client code with empty frequency list
if all(freqs.values()):
return freqs
else:
return None
else:
return None
info['freqs'] = get_freqs
@memoized
def get_orig_capacities():
if target.is_module_available('sched'):
return target.sched.get_capacities(default=1024)
else:
return None
def get_writeable_capacities():
orig_capacities = get_orig_capacities()
if orig_capacities is None:
return None
else:
cpu = 0
path = '/sys/devices/system/cpu/cpu{}/cpu_capacity'.format(cpu)
capa = orig_capacities[cpu]
test_capa = capa - 1 if capa > 1 else capa + 1
try:
target.write_value(path, test_capa, verify=True)
except TargetStableError:
writeable = False
else:
writeable = True
finally:
with contextlib.suppress(TargetStableError):
target.write_value(path, capa)
return writeable
info['cpu-capacities'] = {
'writeable': get_writeable_capacities,
'orig': get_orig_capacities,
}
info['kernel']['symbols-address'] = lambda: DeferredValue(
self._read_kallsyms, target)
def dfs(existing_info, new_info):
def evaluate(existing_info, key, val):
if isinstance(val, Mapping):
return dfs(existing_info[key], val)
else:
if only_missing and key in existing_info:
return None
else:
return val()
return {
key: evaluate(existing_info, key, val)
for key, val in new_info.items()
}
info = dfs(self, info)
return self.add_src(src, info, filter_none=True, **kwargs)
# Internal methods used to compute some keys from a live devlib Target
@classmethod
def _nrg_model_from_target(cls, target):
logger = cls.get_logger()
logger.info('Attempting to read energy model from target')
try:
return EnergyModel.from_target(target)
except (TargetStableError, RuntimeError, ValueError) as err:
logger.error("Couldn't read target energy model: {}".format(err))
return None
@classmethod
def _read_kallsyms(cls, target):
"""
Read and parse the content of ``/proc/kallsyms``.
"""
def parse_line(line):
splitted = re.split(r'\W+', line)
addr = int(splitted[0], base=16)
symtype = splitted[1]
func = splitted[2]
return addr, func
logger = cls.get_logger()
logger.info('Attempting to read kallsyms from target')
try:
with target.revertable_write_value(
'/proc/sys/kernel/kptr_restrict', '0'):
kallsyms = target.read_value('/proc/kallsyms')
except TargetStableError as e:
raise ConfigKeyError("Couldn't read /proc/kallsyms: {}".format(e))
symbols = dict(map(parse_line, kallsyms.splitlines()))
if symbols.keys() == {0}:
raise ConfigKeyError("kallsyms only contains null pointers")
return symbols
class TargetConf(SimpleMultiSrcConf, HideExekallID):
"""
Target connection settings.
Only keys defined below are allowed, with the given meaning and type:
{generated_help}
An instance can be created by calling :class:`~TargetConf` with a
dictionary. The top-level `target-conf` key is not needed here:
.. code-block:: python
TargetConf({{
'name': 'myboard',
'host': 192.0.2.1,
'kind': 'linux',
'username': '******',
'password': '******',
}})
Or alternatively, from a YAML configuration file:
Content of target_conf.yml:
.. literalinclude:: ../target_conf.yml
:language: YAML
::
TargetConf.from_yaml_map('target_conf.yml')
The following special YAML tags can be used in the configuration file:
.. code-block:: YAML
target-conf:
# "!env:<type> ENV_VAR_NAME" can be used to reference an
# environment variable.
name: !env:str BOARD_NAME
port: !env:int PORT
.. note:: That structure in a YAML file is allowed and will work:
* file foo.yml::
target-conf:
name: myboard
* file bar.yml::
target-conf:
!include foo.yml
This will result in that structure which would normally be invalid, but
is handled as a special case::
target-conf:
target-conf:
name: myboard
"""
STRUCTURE = TopLevelKeyDesc('target-conf', 'target connection settings', (
KeyDesc('name', 'Board name, free-form value only used to embelish logs', [str]),
KeyDesc('kind', 'Target kind. Can be "linux" (ssh) or "android" (adb)', [str]),
KeyDesc('host', 'Hostname or IP address of the host', [str, None]),
KeyDesc('username', 'SSH username', [str, None]),
PasswordKeyDesc('password', 'SSH password', [str, None]),
KeyDesc('port', 'SSH or ADB server port', [int, None]),
KeyDesc('device', 'ADB device. Takes precedence over "host"', [str, None]),
KeyDesc('keyfile', 'SSH private key file', [str, None]),
KeyDesc('workdir', 'Remote target workdir', [str]),
KeyDesc('tools', 'List of tools to install on the target', [StrList]),
LevelKeyDesc('wait-boot', 'Wait for the target to finish booting', (
KeyDesc('enable', 'Enable the boot check', [bool]),
KeyDesc('timeout', 'Timeout of the boot check', [int]),
)),
LevelKeyDesc('devlib', 'devlib configuration', (
# Using textual name of the Platform allows this YAML configuration
# to not use any python-specific YAML tags, so TargetConf files can
# be parsed and produced by any other third-party code
LevelKeyDesc('platform', 'devlib.platform.Platform subclass specification', (
KeyDesc('class', 'Name of the class to use', [str]),
KeyDesc('args', 'Keyword arguments to build the Platform object', [Mapping]),
)),
KeyDesc('excluded-modules', 'List of devlib modules to *not* load', [StrList]),
))
))
DEFAULT_SRC = {
'devlib': {
'platform': {
'class': 'devlib.platform.Platform'
}
}
}
class TargetConf(SimpleMultiSrcConf, HideExekallID):
"""
Target connection settings.
Only keys defined below are allowed, with the given meaning and type:
{generated_help}
An instance can be created by calling :class:`~TargetConf` with a
dictionary. The top-level `target-conf` key is not needed here:
.. code-block:: python
TargetConf({{
'name': 'myboard',
'host': 192.0.2.1,
'kind': 'linux',
'username': '******',
'password': '******',
}})
Or alternatively, from a YAML configuration file:
Content of target_conf.yml:
.. literalinclude:: ../target_conf.yml
:language: YAML
::
TargetConf.from_yaml_map('target_conf.yml')
The following special YAML tags can be used in the configuration file:
.. code-block:: YAML
target-conf:
# "!env:<type> ENV_VAR_NAME" can be used to reference an
# environment variable.
name: !env:str BOARD_NAME
port: !env:int PORT
.. note:: Only load trusted YAML files as it can lead to abritrary code
execution.
.. note:: That structure in a YAML file is allowed and will work:
* file foo.yml::
target-conf:
name: myboard
* file bar.yml::
target-conf:
!include foo.yml
This will result in that structure which would normally be invalid, but
is handled as a special case::
target-conf:
target-conf:
name: myboard
"""
STRUCTURE = TopLevelKeyDesc(
'target-conf',
'target connection settings',
(
KeyDesc('name',
'Board name, free-form value only used to embelish logs',
[str]),
KeyDesc('kind',
'Target kind. Can be "linux" (ssh) or "android" (adb)',
[str]),
KeyDesc('host', 'Hostname or IP address of the host', [str, None]),
KeyDesc(
'username',
'SSH username. On ADB connections, "root" username will root adb upon target connection',
[str, None]),
PasswordKeyDesc('password', 'SSH password', [str, None]),
KeyDesc('port', 'SSH or ADB server port', [int, None]),
KeyDesc('device', 'ADB device. Takes precedence over "host"',
[str, None]),
KeyDesc('keyfile', 'SSH private key file', [str, None]),
KeyDesc('strict-host-check',
'Equivalent to StrictHostKeyChecking option of OpenSSH',
[bool, None]),
KeyDesc('workdir', 'Remote target workdir', [str]),
KeyDesc('tools', 'List of tools to install on the target',
[TypedList[str]]),
KeyDesc(
'lazy-platinfo',
'Lazily autodect the platform information to speed up the connection',
[bool]),
LevelKeyDesc('kernel', 'kernel information', (
KeyDesc(
'src',
'Path to kernel source tree matching the kernel running on the target used to build modules',
[str, None]),
LevelKeyDesc('modules', 'kernel modules', (
KeyDesc(
'build-env',
'Environment used to build modules. Can be any of "alpine" (Alpine Linux chroot, recommended) or "host" (host system)',
[str]),
KeyDesc(
'make-variables',
'Extra variables to pass to "make" command, such as "CC"',
[TypedDict[str, object]]),
KeyDesc(
'overlay-backend',
'Backend to use for overlaying folders while building modules. Can be "overlayfs" (overlayfs filesystem, recommended) or "copy (plain folder copy)',
[str]),
)),
)),
LevelKeyDesc(
'wait-boot', 'Wait for the target to finish booting', (
KeyDesc('enable', 'Enable the boot check', [bool]),
KeyDesc('timeout', 'Timeout of the boot check', [int]),
)),
LevelKeyDesc(
'devlib',
'devlib configuration',
(
# Using textual name of the Platform allows this YAML configuration
# to not use any python-specific YAML tags, so TargetConf files can
# be parsed and produced by any other third-party code
LevelKeyDesc(
'platform',
'devlib.platform.Platform subclass specification', (
KeyDesc('class', 'Name of the class to use',
[str]),
KeyDesc(
'args',
'Keyword arguments to build the Platform object',
[Mapping]),
)),
KeyDesc('excluded-modules',
'List of devlib modules to *not* load',
[TypedList[str]]),
KeyDesc(
'file-xfer',
'File transfer method. Can be "sftp" (default) or "scp". (Only valid for linux targets)',
[TypedList[str]]),
))))
DEFAULT_SRC = {
'devlib': {
'platform': {
'class': 'devlib.platform.Platform'
}
}
}
class PlatformInfo(MultiSrcConf, HideExekallID):
"""
Platform-specific information made available to tests.
{generated_help}
"""
# we could use mypy.subtypes.is_subtype and use the infrastructure provided
# by typing module, but adding an external dependency is overkill for what
# we need.
STRUCTURE = TopLevelKeyDesc('platform-info', 'Platform-specific information', (
LevelKeyDesc('rtapp', 'RTapp configuration', (
KeyDesc('calib', 'RTapp calibration dictionary', [IntIntDict]),
)),
LevelKeyDesc('kernel', 'Kernel-related information', (
KeyDesc('version', '', [KernelVersion]),
KernelConfigKeyDesc('config', '', [TypedKernelConfig]),
)),
KeyDesc('nrg-model', 'Energy model object', [EnergyModel]),
KeyDesc('cpu-capacities', 'Dictionary of CPU ID to capacity value', [IntIntDict]),
KeyDesc('abi', 'ABI, e.g. "arm64"', [str]),
KeyDesc('os', 'OS being used, e.g. "linux"', [str]),
KeyDesc('name', 'Free-form name of the board', [str]),
KeyDesc('cpus-count', 'Number of CPUs', [int]),
KeyDesc('freq-domains',
'Frequency domains modeled by a list of CPU IDs for each domain',
[IntListList]),
KeyDesc('freqs', 'Dictionnary of CPU ID to list of frequencies', [IntIntListDict]),
DerivedKeyDesc('capacity-classes',
'Capacity classes modeled by a list of CPU IDs for each ' \
'capacity, sorted by capacity',
[IntListList],
[['cpu-capacities']], compute_capa_classes),
))
"""Some keys have a reserved meaning with an associated type."""
def add_target_src(self,
target,
rta_calib_res_dir,
src='target',
**kwargs):
info = {
'nrg-model': self._nrg_model_from_target(target),
'kernel': {
'version': target.kernel_version,
'config': target.config.typed_config,
},
'abi': target.abi,
'os': target.os,
'rtapp': {
# Since it is expensive to compute, use an on-demand DeferredValue
'calib':
DeferredValue(RTA.get_cpu_calibrations, target,
rta_calib_res_dir)
},
'cpus-count': target.number_of_cpus
}
if hasattr(target, 'cpufreq'):
info['freq-domains'] = list(target.cpufreq.iter_domains())
info['freqs'] = {
cpu: target.cpufreq.list_frequencies(cpu)
for cpu in range(target.number_of_cpus)
}
if hasattr(target, 'sched'):
info['cpu-capacities'] = target.sched.get_capacities(default=1024)
return self.add_src(src, info, filter_none=True, **kwargs)
# Internal methods used to compute some keys from a live devlib Target
@classmethod
def _nrg_model_from_target(cls, target):
logger = cls.get_logger()
logger.info('Attempting to read energy model from target')
try:
return EnergyModel.from_target(target)
except (TargetStableError, RuntimeError, ValueError) as err:
logger.error("Couldn't read target energy model: %s", err)
return None